Examples with DataBlockManager com.tencent.angel.worker.storage.DataBlockManager used on opensource projects

Example 1 with DataBlockManager

use of com.tencent.angel.worker.storage.DataBlockManager in project angel by Tencent.

the class Worker method initAndStart.

/**
 * Init and start Worker's components,run tasks through by {@link TaskManager}
 *
 * @throws Exception the exception
 */
public void initAndStart() throws Exception {
    LOG.info("Init and start worker");
    psAgent = new PSAgent(conf, masterLocation.getIp(), masterLocation.getPort(), workerAttemptId.getWorkerId().getIndex(), false, this);
    dataBlockManager = new DataBlockManager();
    taskManager = new TaskManager();
    LOG.info("Init and start psagent for worker");
    psAgent.initAndStart();
    LOG.info("Init data block manager");
    dataBlockManager.init();
    LOG.info("Init and start worker rpc server");
    workerService = new WorkerService();
    workerService.start();
    LOG.info("Init counter updater");
    counterUpdater.initialize();
    // init task manager and start tasks
    masterClient = psAgent.getMasterClient();
    // start heartbeat thread
    LOG.info("Register to master and start the heartbeat thread");
    startHeartbeatThread();
    LOG.info("Get data splits from master");
    workerGroup = masterClient.getWorkerGroupMetaInfo();
    dataBlockManager.setSplitClassification(workerGroup.getSplits());
    LOG.info("Init and start task manager and all task");
    taskManager.init();
    taskManager.startAllTasks(workerGroup.getWorkerRef(workerAttemptId.getWorkerId()).getTaskIdToContextMap());
    workerInitFinishedFlag.set(true);
}

Example 2 with DataBlockManager

use of com.tencent.angel.worker.storage.DataBlockManager in project angel by Tencent.

the class WorkerTest method testDdataBlocker.

@Test
public void testDdataBlocker() throws Exception {
    try {
        localWorker = LocalClusterContext.get().getWorker(worker0Attempt0Id);
        worker = localWorker.getWorker();
        // test worker get dataBlockManager
        DataBlockManager dataBlockManager = worker.getDataBlockManager();
        assertTrue(dataBlockManager != null);
        assertEquals(dataBlockManager.getSplitClassification(), worker.getWorkerGroup().getSplits());
    } catch (Exception x) {
        LOG.error("run testDdataBlocker failed ", x);
        throw x;
    }
}

Example 3 with DataBlockManager

use of com.tencent.angel.worker.storage.DataBlockManager in project angel by Tencent.

the class TaskContext method getReader.

/**
 * Gets reader.
 *
 * @param <K> key type
 * @param <V> value type
 * @return the reader
 * @throws ClassNotFoundException
 * @throws IOException
 * @throws InterruptedException
 */
@SuppressWarnings("unchecked")
public <K, V> Reader<K, V> getReader() throws ClassNotFoundException, IOException, InterruptedException {
    if (reader == null) {
        DataBlockManager dataBlockManager = WorkerContext.get().getDataBlockManager();
        reader = dataBlockManager.getReader(taskId);
    }
    return reader;
}