Examples with ClockCache com.tencent.angel.psagent.clock.ClockCache used on opensource projects

Example 1 with ClockCache

use of com.tencent.angel.psagent.clock.ClockCache in project angel by Tencent.

the class PSAgent method initAndStart.

public void initAndStart() throws Exception {
    // Get ps locations from master and put them to the location cache.
    locationManager = new PSAgentLocationManager(PSAgentContext.get());
    locationManager.setMasterLocation(masterLocation);
    // Build and initialize rpc client to master
    masterClient = new MasterClient();
    masterClient.init();
    // Build local location
    String localIp = NetUtils.getRealLocalIP();
    int port = NetUtils.chooseAListenPort(conf);
    location = new Location(localIp, port);
    // Initialize matrix meta information
    clockCache = new ClockCache();
    List<MatrixMeta> matrixMetas = masterClient.getMatrices();
    LOG.info("===========================PSAgent get matrices from master," + matrixMetas.size());
    this.matrixMetaManager = new PSAgentMatrixMetaManager(clockCache);
    matrixMetaManager.addMatrices(matrixMetas);
    Map<ParameterServerId, Location> psIdToLocMap = masterClient.getPSLocations();
    List<ParameterServerId> psIds = new ArrayList<>(psIdToLocMap.keySet());
    Collections.sort(psIds, new Comparator<ParameterServerId>() {

        @Override
        public int compare(ParameterServerId s1, ParameterServerId s2) {
            return s1.getIndex() - s2.getIndex();
        }
    });
    int size = psIds.size();
    locationManager.setPsIds(psIds.toArray(new ParameterServerId[0]));
    for (int i = 0; i < size; i++) {
        if (psIdToLocMap.containsKey(psIds.get(i))) {
            locationManager.setPsLocation(psIds.get(i), psIdToLocMap.get(psIds.get(i)));
        }
    }
    matrixTransClient = new MatrixTransportClient();
    matrixClientAdapter = new MatrixClientAdapter();
    opLogCache = new MatrixOpLogCache();
    matrixStorageManager = new MatrixStorageManager();
    matricesCache = new MatricesCache();
    int staleness = conf.getInt(AngelConf.ANGEL_STALENESS, AngelConf.DEFAULT_ANGEL_STALENESS);
    consistencyController = new ConsistencyController(staleness);
    consistencyController.init();
    psAgentInitFinishedFlag.set(true);
    // Start heartbeat thread if need
    if (needHeartBeat) {
        startHeartbeatThread();
    }
    // Start all services
    matrixTransClient.start();
    matrixClientAdapter.start();
    clockCache.start();
    opLogCache.start();
    matricesCache.start();
}

Example 2 with ClockCache

use of com.tencent.angel.psagent.clock.ClockCache in project angel by Tencent.

the class PSAgentTest method testClockCache.

@Test
public void testClockCache() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        ClockCache clockCache = psAgent.getClockCache();
        assertTrue(clockCache != null);
        int rowClock = clockCache.getClock(1, 0);
        assertEquals(rowClock, 0);
    } catch (Exception x) {
        LOG.error("run testClockCache failed ", x);
        throw x;
    }
}

Example 3 with ClockCache

use of com.tencent.angel.psagent.clock.ClockCache in project angel by Tencent.

the class TaskContext method globalSync.

/**
 * Global sync with special matrix,still wait until all matrixes's clock is synchronized.
 *
 * @param matrixId the matrix id
 * @throws InterruptedException
 */
public void globalSync(int matrixId) throws InterruptedException {
    ClockCache clockCache = PSAgentContext.get().getClockCache();
    List<PartitionKey> pkeys = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int syncTimeIntervalMS = PSAgentContext.get().getConf().getInt(AngelConf.ANGEL_PSAGENT_CACHE_SYNC_TIMEINTERVAL_MS, AngelConf.DEFAULT_ANGEL_PSAGENT_CACHE_SYNC_TIMEINTERVAL_MS);
    while (true) {
        boolean sync = true;
        for (PartitionKey pkey : pkeys) {
            if (clockCache.getClock(matrixId, pkey) < getMatrixClock(matrixId)) {
                sync = false;
                break;
            }
        }
        if (!sync) {
            Thread.sleep(syncTimeIntervalMS);
        } else {
            break;
        }
    }
}

Example 4 with ClockCache

use of com.tencent.angel.psagent.clock.ClockCache in project angel by Tencent.

the class TaskContext method getPSMatrixClock.

/**
 * Get the clock value of a matrix
 * @param matrixId matrix id
 * @return clock value
 */
public int getPSMatrixClock(int matrixId) {
    ClockCache clockCache = PSAgentContext.get().getClockCache();
    List<PartitionKey> pkeys = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId);
    int size = pkeys.size();
    int clock = Integer.MAX_VALUE;
    int partClock = 0;
    for (int i = 0; i < size; i++) {
        partClock = clockCache.getClock(matrixId, pkeys.get(i));
        if (partClock < clock) {
            clock = partClock;
        }
    }
    return clock;
}