Examples with BlockMaster alluxio.master.block.BlockMaster used on opensource projects

Example 6 with BlockMaster

use of alluxio.master.block.BlockMaster in project alluxio by Alluxio.

the class MasterTestUtils method createStandbyFileSystemMasterFromJournal.

/**
   * Creates a new standby {@link FileSystemMaster} from journal.
   *
   * @return a new FileSystemMaster
   * @throws IOException
   */
public static FileSystemMaster createStandbyFileSystemMasterFromJournal() throws IOException {
    String masterJournal = Configuration.get(PropertyKey.MASTER_JOURNAL_FOLDER);
    JournalFactory journalFactory = new JournalFactory.ReadWrite(masterJournal);
    BlockMaster blockMaster = new BlockMaster(journalFactory);
    FileSystemMaster fsMaster = new FileSystemMaster(blockMaster, journalFactory);
    blockMaster.start(false);
    fsMaster.start(false);
    return fsMaster;
}

Example 7 with BlockMaster

use of alluxio.master.block.BlockMaster in project alluxio by Alluxio.

the class MasterFaultToleranceIntegrationTest method failoverWorkerRegister.

@Test
public void failoverWorkerRegister() throws Exception {
    // Stop the default cluster.
    after();
    // Create a new cluster, with no workers initially
    final MultiMasterLocalAlluxioCluster cluster = new MultiMasterLocalAlluxioCluster(2, 0);
    cluster.initConfiguration();
    cluster.start();
    try {
        // Get the first block master
        BlockMaster blockMaster1 = cluster.getMaster().getInternalMaster().getBlockMaster();
        // Register worker 1
        long workerId1a = blockMaster1.getWorkerId(new alluxio.wire.WorkerNetAddress().setHost("host1"));
        blockMaster1.workerRegister(workerId1a, Collections.EMPTY_LIST, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP);
        // Register worker 2
        long workerId2a = blockMaster1.getWorkerId(new alluxio.wire.WorkerNetAddress().setHost("host2"));
        blockMaster1.workerRegister(workerId2a, Collections.EMPTY_LIST, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP);
        Assert.assertEquals(2, blockMaster1.getWorkerCount());
        // Worker heartbeats should return "Nothing"
        Assert.assertEquals(CommandType.Nothing, blockMaster1.workerHeartbeat(workerId1a, Collections.EMPTY_MAP, Collections.EMPTY_LIST, Collections.EMPTY_MAP).getCommandType());
        Assert.assertEquals(CommandType.Nothing, blockMaster1.workerHeartbeat(workerId2a, Collections.EMPTY_MAP, Collections.EMPTY_LIST, Collections.EMPTY_MAP).getCommandType());
        Assert.assertTrue(cluster.stopLeader());
        cluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
        // Get the new block master, after the failover
        BlockMaster blockMaster2 = cluster.getMaster().getInternalMaster().getBlockMaster();
        // Worker 2 tries to heartbeat (with original id), and should get "Register" in response.
        Assert.assertEquals(CommandType.Register, blockMaster2.workerHeartbeat(workerId2a, Collections.EMPTY_MAP, Collections.EMPTY_LIST, Collections.EMPTY_MAP).getCommandType());
        // Worker 2 re-registers (and gets a new worker id)
        long workerId2b = blockMaster2.getWorkerId(new alluxio.wire.WorkerNetAddress().setHost("host2"));
        blockMaster2.workerRegister(workerId2b, Collections.EMPTY_LIST, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP);
        // Worker 1 tries to heartbeat (with original id), and should get "Register" in response.
        Assert.assertEquals(CommandType.Register, blockMaster2.workerHeartbeat(workerId1a, Collections.EMPTY_MAP, Collections.EMPTY_LIST, Collections.EMPTY_MAP).getCommandType());
        // Worker 1 re-registers (and gets a new worker id)
        long workerId1b = blockMaster2.getWorkerId(new alluxio.wire.WorkerNetAddress().setHost("host1"));
        blockMaster2.workerRegister(workerId1b, Collections.EMPTY_LIST, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP);
    } finally {
        cluster.stop();
    }
    // Start the default cluster.
    before();
}

Example 8 with BlockMaster

use of alluxio.master.block.BlockMaster in project alluxio by Alluxio.

the class FileSystemMasterIntegrationTest method getCapacityBytes.

@Test
public void getCapacityBytes() {
    BlockMaster blockMaster = mLocalAlluxioClusterResource.get().getMaster().getInternalMaster().getBlockMaster();
    Assert.assertEquals(1000, blockMaster.getCapacityBytes());
}

Example 9 with BlockMaster

use of alluxio.master.block.BlockMaster in project alluxio by Alluxio.

the class FreeAndDeleteIntegrationTest method freeAndDeleteIntegration.

@Test
public void freeAndDeleteIntegration() throws Exception {
    HeartbeatScheduler.await(HeartbeatContext.WORKER_BLOCK_SYNC, 5, TimeUnit.SECONDS);
    HeartbeatScheduler.await(HeartbeatContext.MASTER_LOST_FILES_DETECTION, 5, TimeUnit.SECONDS);
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, mWriteBoth);
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    final Long blockId = status.getBlockIds().get(0);
    BlockMaster bm = mLocalAlluxioClusterResource.get().getMaster().getInternalMaster().getBlockMaster();
    BlockInfo blockInfo = bm.getBlockInfo(blockId);
    Assert.assertEquals(2, blockInfo.getLength());
    Assert.assertFalse(blockInfo.getLocations().isEmpty());
    final BlockWorker bw = mLocalAlluxioClusterResource.get().getWorker().getBlockWorker();
    Assert.assertTrue(bw.hasBlockMeta(blockId));
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
    mFileSystem.free(filePath);
    IntegrationTestUtils.waitForBlocksToBeFreed(bw, blockId);
    status = mFileSystem.getStatus(filePath);
    // Verify block metadata in master is still present after block freed.
    Assert.assertEquals(1, status.getBlockIds().size());
    blockInfo = bm.getBlockInfo(status.getBlockIds().get(0));
    Assert.assertEquals(2, blockInfo.getLength());
    // Verify the block has been removed from all workers.
    Assert.assertTrue(blockInfo.getLocations().isEmpty());
    Assert.assertFalse(bw.hasBlockMeta(blockId));
    // Verify the removed block is added to LostBlocks list.
    Assert.assertTrue(bm.getLostBlocks().contains(blockInfo.getBlockId()));
    mFileSystem.delete(filePath);
    try {
        // File is immediately gone after delete.
        mFileSystem.getStatus(filePath);
        Assert.fail(String.format("Expected file %s being deleted but it was not.", filePath));
    } catch (FileDoesNotExistException e) {
    // expected
    }
    // Execute the lost files detection.
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_LOST_FILES_DETECTION);
    // Verify the blocks are not in mLostBlocks.
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
}

Example 10 with BlockMaster

use of alluxio.master.block.BlockMaster in project alluxio by Alluxio.

the class FileSystemMasterTest method startServices.

private void startServices() throws Exception {
    JournalFactory journalFactory = new JournalFactory.ReadWrite(mJournalFolder);
    mBlockMaster = new BlockMaster(journalFactory);
    mExecutorService = Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("FileSystemMasterTest-%d", true));
    mFileSystemMaster = new FileSystemMaster(mBlockMaster, journalFactory, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mBlockMaster.start(true);
    mFileSystemMaster.start(true);
    // set up workers
    mWorkerId1 = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost").setRpcPort(80).setDataPort(81).setWebPort(82));
    mBlockMaster.workerRegister(mWorkerId1, Arrays.asList("MEM", "SSD"), ImmutableMap.of("MEM", (long) Constants.MB, "SSD", (long) Constants.MB), ImmutableMap.of("MEM", (long) Constants.KB, "SSD", (long) Constants.KB), new HashMap<String, List<Long>>());
    mWorkerId2 = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("remote").setRpcPort(80).setDataPort(81).setWebPort(82));
    mBlockMaster.workerRegister(mWorkerId2, Arrays.asList("MEM", "SSD"), ImmutableMap.of("MEM", (long) Constants.MB, "SSD", (long) Constants.MB), ImmutableMap.of("MEM", (long) Constants.KB, "SSD", (long) Constants.KB), new HashMap<String, List<Long>>());
}