Examples with HeartbeatThread alluxio.heartbeat.HeartbeatThread used on opensource projects

Example 1 with HeartbeatThread

use of alluxio.heartbeat.HeartbeatThread in project alluxio by Alluxio.

the class DefaultBlockWorker method start.

/**
   * Runs the block worker. The thread must be called after all services (e.g., web, dataserver)
   * started.
   *
   * @throws IOException if a non-Alluxio related exception occurs
   */
@Override
public void start() throws IOException {
    Preconditions.checkNotNull(mWorkerId, "mWorkerId");
    Preconditions.checkNotNull(mAddress, "mAddress");
    // Setup BlockMasterSync
    mBlockMasterSync = new BlockMasterSync(this, mWorkerId, mAddress, mBlockMasterClient);
    // Setup PinListSyncer
    mPinListSync = new PinListSync(this, mFileSystemMasterClient);
    // Setup session cleaner
    mSessionCleaner = new SessionCleaner(new SessionCleanupCallback() {

        /**
       * Cleans up after sessions, to prevent zombie sessions holding local resources.
       */
        @Override
        public void cleanupSessions() {
            for (long session : mSessions.getTimedOutSessions()) {
                mSessions.removeSession(session);
                mBlockStore.cleanupSession(session);
                mUnderFileSystemBlockStore.cleanupSession(session);
            }
        }
    });
    // Setup space reserver
    if (Configuration.getBoolean(PropertyKey.WORKER_TIERED_STORE_RESERVER_ENABLED)) {
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.WORKER_SPACE_RESERVER, new SpaceReserver(this), Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_RESERVER_INTERVAL_MS)));
    }
    getExecutorService().submit(new HeartbeatThread(HeartbeatContext.WORKER_BLOCK_SYNC, mBlockMasterSync, Configuration.getInt(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS)));
    // Start the pinlist syncer to perform the periodical fetching
    getExecutorService().submit(new HeartbeatThread(HeartbeatContext.WORKER_PIN_LIST_SYNC, mPinListSync, Configuration.getInt(PropertyKey.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS)));
    // Start the session cleanup checker to perform the periodical checking
    getExecutorService().submit(mSessionCleaner);
}

Example 2 with HeartbeatThread

use of alluxio.heartbeat.HeartbeatThread in project alluxio by Alluxio.

the class SpaceReserverTest method reserveCorrectAmountsOfSpace.

@Test
public void reserveCorrectAmountsOfSpace() throws Exception {
    BlockWorker blockWorker = PowerMockito.mock(BlockWorker.class);
    BlockStoreMeta storeMeta = PowerMockito.mock(BlockStoreMeta.class);
    Mockito.when(blockWorker.getStoreMeta()).thenReturn(storeMeta);
    Map<String, Long> capacityBytesOnTiers = ImmutableMap.of("MEM", 400L, "HDD", 1000L);
    Mockito.when(storeMeta.getCapacityBytesOnTiers()).thenReturn(capacityBytesOnTiers);
    // Create two tiers named "MEM" and "HDD" with aliases 0 and 1.
    TieredBlockStoreTestUtils.setupConfWithMultiTier("/", new int[] { 0, 1 }, new String[] { "MEM", "HDD" }, new String[][] { new String[] { "/a" }, new String[] { "/b" } }, new long[][] { new long[] { 0 }, new long[] { 0 } }, "/");
    PropertyKey reserveRatioProp = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(0);
    Configuration.set(reserveRatioProp, "0.2");
    reserveRatioProp = PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(1);
    Configuration.set(reserveRatioProp, "0.3");
    SpaceReserver spaceReserver = new SpaceReserver(blockWorker);
    mExecutorService.submit(new HeartbeatThread(HeartbeatContext.WORKER_SPACE_RESERVER, spaceReserver, 0));
    // Run the space reserver once.
    HeartbeatScheduler.execute(HeartbeatContext.WORKER_SPACE_RESERVER);
    // 400 * 0.2 = 80
    Mockito.verify(blockWorker).freeSpace(Sessions.MIGRATE_DATA_SESSION_ID, 80L, "MEM");
    // 400 * 0.2 + 1000 * 0.3 = 380
    Mockito.verify(blockWorker).freeSpace(Sessions.MIGRATE_DATA_SESSION_ID, 380L, "HDD");
}

Example 3 with HeartbeatThread

use of alluxio.heartbeat.HeartbeatThread in project alluxio by Alluxio.

the class DefaultBlockMaster method start.

@Override
public void start(Boolean isLeader) throws IOException {
    super.start(isLeader);
    if (isLeader) {
        mLostWorkerDetectionService = getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOST_WORKER_DETECTION, new LostWorkerDetectionHeartbeatExecutor(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_LOST_WORKER_DETECTION_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
    }
    // This periodically scans all open register streams and closes hanging ones
    getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_WORKER_REGISTER_SESSION_CLEANER, new WorkerRegisterStreamGCExecutor(), (int) ServerConfiguration.global().getMs(PropertyKey.MASTER_WORKER_REGISTER_STREAM_RESPONSE_TIMEOUT), ServerConfiguration.global(), mMasterContext.getUserState()));
}

Example 4 with HeartbeatThread

use of alluxio.heartbeat.HeartbeatThread in project alluxio by Alluxio.

the class ActiveSyncManager method launchPollingThread.

/**
 * Launches polling thread on a particular mount point with starting txId.
 *
 * @param mountId launch polling thread on a mount id
 * @param txId specifies the transaction id to initialize the pollling thread
 */
public void launchPollingThread(long mountId, long txId) {
    LOG.debug("launch polling thread for mount id {}, txId {}", mountId, txId);
    if (!mPollerMap.containsKey(mountId)) {
        UfsManager.UfsClient ufsClient = mMountTable.getUfsClient(mountId);
        if (ufsClient == null) {
            LOG.warn("Mount id {} does not exist", mountId);
            return;
        }
        try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) {
            ufsResource.get().startActiveSyncPolling(txId);
        } catch (IOException e) {
            LOG.warn("IO Exception trying to launch Polling thread: {}", e.toString());
        }
        ActiveSyncer syncer = new ActiveSyncer(mFileSystemMaster, this, mMountTable, mountId);
        Future<?> future = getExecutor().submit(new HeartbeatThread(HeartbeatContext.MASTER_ACTIVE_UFS_SYNC, syncer, (int) ServerConfiguration.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INTERVAL), ServerConfiguration.global(), ServerUserState.global()));
        mPollerMap.put(mountId, future);
    }
}

Example 5 with HeartbeatThread

use of alluxio.heartbeat.HeartbeatThread in project alluxio by Alluxio.

the class DefaultMetaMaster method start.

@Override
public void start(Boolean isPrimary) throws IOException {
    super.start(isPrimary);
    mWorkerConfigStore.reset();
    mMasterConfigStore.reset();
    if (isPrimary) {
        // Add the configuration of the current leader master
        mMasterConfigStore.registerNewConf(mMasterAddress, ConfigurationUtils.getConfiguration(ServerConfiguration.global(), Scope.MASTER));
        // The service that detects lost standby master nodes
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOST_MASTER_DETECTION, new LostMasterDetectionHeartbeatExecutor(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOG_CONFIG_REPORT_SCHEDULING, new LogConfigReportHeartbeatExecutor(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_LOG_CONFIG_REPORT_HEARTBEAT_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_DAILY_BACKUP_ENABLED)) {
            mDailyBackup = new DailyMetadataBackup(this, Executors.newSingleThreadScheduledExecutor(ThreadFactoryUtils.build("DailyMetadataBackup-%d", true)), mUfsManager);
            mDailyBackup.start();
        }
        if (mJournalSpaceMonitor != null) {
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_JOURNAL_SPACE_MONITOR, mJournalSpaceMonitor, ServerConfiguration.getMs(PropertyKey.MASTER_JOURNAL_SPACE_MONITOR_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        if (mState.getClusterID().equals(INVALID_CLUSTER_ID)) {
            try (JournalContext context = createJournalContext()) {
                String clusterID = java.util.UUID.randomUUID().toString();
                mState.applyAndJournal(context, clusterID);
                LOG.info("Created new cluster ID {}", clusterID);
            }
            if (ServerConfiguration.getBoolean(PropertyKey.MASTER_UPDATE_CHECK_ENABLED) && !ServerConfiguration.getBoolean(PropertyKey.TEST_MODE)) {
                getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UPDATE_CHECK, new UpdateChecker(this), (int) ServerConfiguration.getMs(PropertyKey.MASTER_UPDATE_CHECK_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
            }
        } else {
            LOG.info("Detected existing cluster ID {}", mState.getClusterID());
        }
        mBackupRole = new BackupLeaderRole(mCoreMasterContext);
    } else {
        if (ConfigurationUtils.isHaMode(ServerConfiguration.global())) {
            // Standby master should setup MetaMasterSync to communicate with the leader master
            RetryHandlingMetaMasterMasterClient metaMasterClient = new RetryHandlingMetaMasterMasterClient(MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build());
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.META_MASTER_SYNC, new MetaMasterSync(mMasterAddress, metaMasterClient), (int) ServerConfiguration.getMs(PropertyKey.MASTER_STANDBY_HEARTBEAT_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
            LOG.info("Standby master with address {} starts sending heartbeat to leader master.", mMasterAddress);
        }
        // Enable worker role if backup delegation is enabled.
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_BACKUP_DELEGATION_ENABLED)) {
            mBackupRole = new BackupWorkerRole(mCoreMasterContext);
        }
    }
}