Examples with WorkerManager com.tencent.angel.master.worker.WorkerManager used on opensource projects

Example 6 with WorkerManager

use of com.tencent.angel.master.worker.WorkerManager in project angel by Tencent.

the class PSAgentTest method testConsistencyController.

@Test
public void testConsistencyController() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        // ConsistencyController consistControl = psAgent.getConsistencyController();
        // assertTrue(consistControl != null);
        PSAgentContext psAgentContext = PSAgentContext.get();
        assertTrue(psAgentContext.getPsAgent() != null);
        TaskContext taskContext1 = psAgentContext.getTaskContext(1);
        TaskContext taskContext2 = psAgentContext.getTaskContext(2);
        assertTrue(taskContext1 != null);
        assertTrue(taskContext2 != null);
    } catch (Exception x) {
        LOG.error("run testConsistencyController failed ", x);
        throw x;
    }
}

Example 7 with WorkerManager

use of com.tencent.angel.master.worker.WorkerManager in project angel by Tencent.

the class PSAgentTest method testLocationCache.

@Test
public void testLocationCache() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        PSAgentLocationManager locationCache = psAgent.getLocationManager();
        assertTrue(locationCache != null);
        // test master location
        Location masterLoc = locationCache.getMasterLocation();
        String ipRegex = "(2[5][0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})";
        Pattern pattern = Pattern.compile(ipRegex);
        Matcher matcher = pattern.matcher(masterLoc.getIp());
        assertTrue(matcher.matches());
        assertTrue(masterLoc.getPort() >= 1 && masterLoc.getPort() <= 65535);
        // test ps location
        Location psLoc = locationCache.getPsLocation(psId);
        matcher = pattern.matcher(psLoc.getIp());
        assertTrue(matcher.matches());
        assertTrue(psLoc.getPort() >= 1 && psLoc.getPort() <= 65535);
        // assertEquals(psLoc, locationCache.updateAndGetPSLocation(psId));
        // test all ps ids
        ParameterServerId[] allPSIds = locationCache.getPsIds();
        assertEquals(allPSIds.length, 1);
        assertEquals(allPSIds[0], psId);
    } catch (Exception x) {
        LOG.error("run testLocationCache failed ", x);
        throw x;
    }
}

Example 8 with WorkerManager

use of com.tencent.angel.master.worker.WorkerManager in project angel by Tencent.

the class PSAgentTest method testPSClient.

@Test
public void testPSClient() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        // psAgent.initAndStart();
        // test conf
        Configuration conf = psAgent.getConf();
        assertTrue(conf != null);
        assertEquals(conf.get(AngelConf.ANGEL_DEPLOY_MODE), "LOCAL");
        // test master location
        Location masterLoc = psAgent.getMasterLocation();
        String ipRegex = "(2[5][0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})";
        Pattern pattern = Pattern.compile(ipRegex);
        Matcher matcher = pattern.matcher(masterLoc.getIp());
        assertTrue(matcher.matches());
        assertTrue(masterLoc.getPort() >= 1 && masterLoc.getPort() <= 65535);
        // test app id
        ApplicationId appId = psAgent.getAppId();
        // test user
        String user = psAgent.getUser();
        // test ps agent attempt id
        int psAgentId = psAgent.getId();
        assertEquals(psAgentId, 1);
        // test connection
        TConnection conn = psAgent.getConnection();
        assertTrue(conn != null);
        // test master client
        MasterClient masterClient = psAgent.getMasterClient();
        assertTrue(masterClient != null);
        // test ip
        String ip = psAgent.getIp();
        matcher = pattern.matcher(ip);
        assertTrue(matcher.matches());
        // test loc
        Location loc = psAgent.getLocation();
        assertTrue(loc != null);
        matcher = pattern.matcher(loc.getIp());
        assertTrue(matcher.matches());
        assertTrue(loc.getPort() >= 1 && loc.getPort() <= 65535);
    } catch (Exception x) {
        LOG.error("run testPSClient failed ", x);
        throw x;
    }
}

Example 9 with WorkerManager

use of com.tencent.angel.master.worker.WorkerManager in project angel by Tencent.

the class PSAgentTest method testMatrixLocationManager.

@Test
public void testMatrixLocationManager() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        PSAgentMatrixMetaManager matrixPartitionRouter = psAgent.getMatrixMetaManager();
        PSAgentLocationManager locationCache = psAgent.getLocationManager();
        assertTrue(matrixPartitionRouter != null);
        // test ps location
        Location psLoc = locationCache.getPsLocation(psId);
        String ipRegex = "(2[5][0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})";
        Pattern pattern = Pattern.compile(ipRegex);
        Matcher matcher = pattern.matcher(psLoc.getIp());
        assertTrue(matcher.matches());
        assertTrue(psLoc.getPort() >= 1 && psLoc.getPort() <= 65535);
        int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
        int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
        // test partitions
        List<PartitionKey> partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id);
        assertEquals(partition1Keys.size(), 2);
        List<PartitionKey> partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id);
        assertEquals(partition2Keys.size(), 2);
        partition1Keys.clear();
        partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
        assertEquals(partition1Keys.size(), 2);
        partition2Keys.clear();
        partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
        assertEquals(partition2Keys.size(), 2);
        int rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
        assertEquals(rowPartSize, 2);
        rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
        assertEquals(rowPartSize, 2);
    } catch (Exception x) {
        LOG.error("run testMatrixLocationManager failed ", x);
        throw x;
    }
}

Example 10 with WorkerManager

use of com.tencent.angel.master.worker.WorkerManager in project angel by Tencent.

the class PSAgentTest method testMatrixMetaManager.

@Test
public void testMatrixMetaManager() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        PSAgentMatrixMetaManager matrixMetaManager = psAgent.getMatrixMetaManager();
        assertTrue(matrixMetaManager != null);
        // test all matrix ids
        assertEquals(matrixMetaManager.getMatrixMetas().size(), 2);
        // test all matrix names
        assertNotNull(matrixMetaManager.getMatrixMeta("w1"));
        assertNotNull(matrixMetaManager.getMatrixMeta("w2"));
        // test matrix attribute
        int matrixId1 = matrixMetaManager.getMatrixId("w1");
        int matrixId2 = matrixMetaManager.getMatrixId("w2");
        String hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId1).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
        assertEquals(hogwildAttr, "true");
        hogwildAttr = matrixMetaManager.getMatrixMeta(matrixId2).getAttribute(MatrixConf.MATRIX_HOGWILD, "true");
        assertEquals(hogwildAttr, "true");
        int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
        int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
        // test matrix meta
        MatrixMeta matrixMetaById = matrixMetaManager.getMatrixMeta(matrix1Id);
        MatrixMeta matrixMetaByName = matrixMetaManager.getMatrixMeta("w1");
        assertEquals(matrixMetaById, matrixMetaByName);
        assertEquals(matrixMetaById.getName(), "w1");
        assertEquals(matrixMetaById.getRowNum(), 1);
        assertEquals(matrixMetaById.getColNum(), 100000);
        assertEquals(matrixMetaById.getRowType(), RowType.T_DOUBLE_DENSE);
        assertEquals(matrixMetaById.getAttribute(MatrixConf.MATRIX_HOGWILD, "true"), "true");
        assertEquals(matrixMetaById.getStaleness(), 0);
    } catch (Exception x) {
        LOG.error("run testMatrixMetaManager failed ", x);
        throw x;
    }
}