Examples with DefaultBlockMaster alluxio.master.block.DefaultBlockMaster used on opensource projects

Example 1 with DefaultBlockMaster

use of alluxio.master.block.DefaultBlockMaster in project alluxio by Alluxio.

the class AbstractLocalAlluxioCluster method stopWorkers.

/**
 * Stops the workers.
 */
public void stopWorkers() throws Exception {
    if (mWorkers == null) {
        return;
    }
    for (WorkerProcess worker : mWorkers) {
        worker.stop();
    }
    for (Thread thread : mWorkerThreads) {
        while (thread.isAlive()) {
            LOG.info("Stopping thread {}.", thread.getName());
            thread.interrupt();
            thread.join(1000);
        }
    }
    mWorkerThreads.clear();
    // forget all the workers in the master
    LocalAlluxioMaster master = getLocalAlluxioMaster();
    if (master != null) {
        DefaultBlockMaster bm = (DefaultBlockMaster) master.getMasterProcess().getMaster(BlockMaster.class);
        bm.forgetAllWorkers();
    }
}

Example 2 with DefaultBlockMaster

use of alluxio.master.block.DefaultBlockMaster in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method registerLostWorker.

@Test
public // The master has marked the worker as lost.
void registerLostWorker() throws Exception {
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    // The worker registers to the master
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(workerId);
    prepareBlocksOnMaster(requestChunks);
    Queue<Throwable> errorQueue = new ConcurrentLinkedQueue<>();
    sendStreamToMaster(requestChunks, RegisterStreamTestUtils.getErrorCapturingResponseObserver(errorQueue));
    // Verify the worker has been registered
    assertEquals(0, errorQueue.size());
    assertEquals(1, mBlockMaster.getWorkerCount());
    // The worker has lost heartbeat and been forgotten
    MasterWorkerInfo worker = mBlockMaster.getWorker(workerId);
    long newTimeMs = worker.getLastUpdatedTimeMs() + MASTER_WORKER_TIMEOUT + 1;
    mClock.setTimeMs(newTimeMs);
    DefaultBlockMaster.LostWorkerDetectionHeartbeatExecutor lostWorkerDetector = ((DefaultBlockMaster) mBlockMaster).new LostWorkerDetectionHeartbeatExecutor();
    lostWorkerDetector.heartbeat();
    // Verify the worker has been forgotten
    assertEquals(0, mBlockMaster.getWorkerCount());
    // Register again
    Queue<Throwable> newErrorQueue = new ConcurrentLinkedQueue<>();
    sendStreamToMaster(requestChunks, RegisterStreamTestUtils.getErrorCapturingResponseObserver(newErrorQueue));
    // Verify the worker is registered again
    assertEquals(0, errorQueue.size());
    MasterWorkerInfo updatedWorker = mBlockMaster.getWorker(workerId);
    assertEquals(TIER_BLOCK_TOTAL, updatedWorker.getBlockCount());
    assertEquals(0, updatedWorker.getToRemoveBlockCount());
    assertEquals(1, mBlockMaster.getWorkerCount());
    // Verify the worker is readable and writable
    verifyWorkerWritable(workerId);
}

Example 3 with DefaultBlockMaster

use of alluxio.master.block.DefaultBlockMaster in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method before.

/**
 * Sets up the dependencies before a test runs.
 */
@Before
public void before() throws Exception {
    // Set the config properties
    ServerConfiguration.set(PropertyKey.WORKER_REGISTER_STREAM_ENABLED, true);
    ServerConfiguration.set(PropertyKey.WORKER_REGISTER_STREAM_BATCH_SIZE, BATCH_SIZE);
    ServerConfiguration.set(PropertyKey.MASTER_WORKER_TIMEOUT_MS, MASTER_WORKER_TIMEOUT);
    ServerConfiguration.set(PropertyKey.MASTER_WORKER_REGISTER_STREAM_RESPONSE_TIMEOUT, "1s");
    ServerConfiguration.set(PropertyKey.MASTER_WORKER_REGISTER_LEASE_ENABLED, false);
    mRegistry = new MasterRegistry();
    mMasterContext = MasterTestUtils.testMasterContext();
    mMetricsMaster = new MetricsMasterFactory().create(mRegistry, mMasterContext);
    mRegistry.add(MetricsMaster.class, mMetricsMaster);
    mClock = new ManualClock();
    mExecutorService = Executors.newFixedThreadPool(10, ThreadFactoryUtils.build("TestBlockMaster-%d", true));
    mBlockMaster = new DefaultBlockMaster(mMetricsMaster, mMasterContext, mClock, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mRegistry.add(BlockMaster.class, mBlockMaster);
    mRegistry.start(true);
    mHandler = new BlockMasterWorkerServiceHandler(mBlockMaster);
}

Example 4 with DefaultBlockMaster

use of alluxio.master.block.DefaultBlockMaster in project alluxio by Alluxio.

the class BackupManagerTest method rocksInodeStoreIteratorNotUsed.

@Test
public void rocksInodeStoreIteratorNotUsed() throws Exception {
    // Prepare some data for the iterator
    List<InodeView> inodes = new ArrayList<>();
    inodes.add(createRootDir(99L));
    inodes.add(createNewFile(100L));
    inodes.add(createNewFile(101L));
    inodes.add(createNewFile(102L));
    // When RocksInodeStore.iterator(), return mock iterator
    RocksInodeStore mockInodeStore = mock(RocksInodeStore.class);
    // Make sure the iterator is not used in the backup operation
    when(mockInodeStore.getCloseableIterator()).thenThrow(new UnsupportedOperationException());
    // When the root inode is asked for, return the directory
    InodeView dir = inodes.get(0);
    when(mockInodeStore.get(eq(0L))).thenReturn(Optional.of(new InodeDirectory((InodeDirectoryView) dir)));
    CoreMasterContext masterContext = MasterTestUtils.testMasterContext(new NoopJournalSystem(), null, () -> new HeapBlockStore(), x -> mockInodeStore);
    mMetricsMaster = new MetricsMasterFactory().create(mRegistry, masterContext);
    mBlockMaster = new DefaultBlockMaster(mMetricsMaster, masterContext, mClock, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mRegistry.add(BlockMaster.class, mBlockMaster);
    // Prepare the FileSystemMaster for the backup operation
    FileSystemMaster fsMaster = new DefaultFileSystemMaster(mBlockMaster, masterContext, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mRegistry.add(FileSystemMaster.class, fsMaster);
    mRegistry.start(true);
    // Finish backup operation
    BackupManager manager = new BackupManager(mRegistry);
    File backupDir = AlluxioTestDirectory.createTemporaryDirectory("backup-dir");
    File backupFile = new File(backupDir, "1.backup");
    AtomicLong counter = new AtomicLong(0L);
    // No exception means the RocksInodeStore iterator is not used
    manager.backup(new FileOutputStream(backupFile), counter);
}

Example 5 with DefaultBlockMaster

use of alluxio.master.block.DefaultBlockMaster in project alluxio by Alluxio.

the class BackupManagerTest method rocksBlockStoreIteratorClosed.

@Test
public void rocksBlockStoreIteratorClosed() throws Exception {
    // Prepare some data for the iterator
    List<BlockStore.Block> blocks = new ArrayList<>();
    blocks.add(createNewBlock(1L));
    blocks.add(createNewBlock(2L));
    blocks.add(createNewBlock(3L));
    // When RocksBlockStore.iterator(), return mock iterator
    AtomicBoolean blockIteratorClosed = new AtomicBoolean(false);
    CloseableIterator<BlockStore.Block> testBlockIter = CloseableIterator.create(blocks.iterator(), (whatever) -> blockIteratorClosed.set(true));
    RocksBlockStore mockBlockStore = mock(RocksBlockStore.class);
    when(mockBlockStore.iterator()).thenReturn(testBlockIter);
    // Prepare the BlockMaster for the backup operation
    CoreMasterContext masterContext = MasterTestUtils.testMasterContext(new NoopJournalSystem(), null, () -> mockBlockStore, x -> new HeapInodeStore());
    mMetricsMaster = new MetricsMasterFactory().create(mRegistry, masterContext);
    mBlockMaster = new DefaultBlockMaster(mMetricsMaster, masterContext, mClock, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mRegistry.add(BlockMaster.class, mBlockMaster);
    mRegistry.start(true);
    // Perform the backup operation
    BackupManager manager = new BackupManager(mRegistry);
    File backupDir = AlluxioTestDirectory.createTemporaryDirectory("backup-dir");
    File backupFile = new File(backupDir, "1.backup");
    AtomicLong counter = new AtomicLong(0L);
    manager.backup(new FileOutputStream(backupFile), counter);
    // verify iterators all closed properly
    assertTrue(blockIteratorClosed.get());
}