Examples with BlockWorker alluxio.worker.block.BlockWorker used on opensource projects

Example 1 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class BlockMasterClientRestApiTest method getBlockInfo.

@Test
public void getBlockInfo() throws Exception {
    long sessionId = 1;
    long blockId = 2;
    String tierAlias = "MEM";
    long initialBytes = 3;
    BlockWorker blockWorker = mResource.get().getWorker().getBlockWorker();
    String file = blockWorker.createBlock(sessionId, blockId, tierAlias, initialBytes);
    FileOutputStream outStream = new FileOutputStream(file);
    outStream.write("abc".getBytes());
    outStream.close();
    blockWorker.commitBlock(sessionId, blockId);
    Map<String, String> params = new HashMap<>();
    params.put("blockId", Long.toString(blockId));
    String response = new TestCase(mHostname, mPort, getEndpoint(BlockMasterClientRestServiceHandler.GET_BLOCK_INFO), params, HttpMethod.GET, null).call();
    BlockInfo blockInfo = new ObjectMapper().readValue(response, BlockInfo.class);
    Assert.assertEquals(blockId, blockInfo.getBlockId());
    Assert.assertEquals(initialBytes, blockInfo.getLength());
    Assert.assertEquals("MEM", Iterables.getOnlyElement(blockInfo.getLocations()).getTierAlias());
}

Example 2 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class KeyValueWorkerFactory method create.

@Override
public KeyValueWorker create(List<? extends Worker> workers) {
    if (!Configuration.getBoolean(PropertyKey.KEY_VALUE_ENABLED)) {
        return null;
    }
    LOG.info("Creating {} ", KeyValueWorker.class.getName());
    for (Worker worker : workers) {
        if (worker instanceof BlockWorker) {
            LOG.info("{} is created", KeyValueWorker.class.getName());
            return new KeyValueWorker(((BlockWorker) worker));
        }
    }
    LOG.error("Fail to create {} due to missing {}", KeyValueWorker.class.getName(), BlockWorker.class.getName());
    return null;
}

Example 3 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class FreeAndDeleteIntegrationTest method freeAndDeleteIntegration.

@Test
public void freeAndDeleteIntegration() throws Exception {
    HeartbeatScheduler.await(HeartbeatContext.WORKER_BLOCK_SYNC, 5, TimeUnit.SECONDS);
    HeartbeatScheduler.await(HeartbeatContext.MASTER_LOST_FILES_DETECTION, 5, TimeUnit.SECONDS);
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, mWriteBoth);
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    final Long blockId = status.getBlockIds().get(0);
    BlockMaster bm = mLocalAlluxioClusterResource.get().getMaster().getInternalMaster().getBlockMaster();
    BlockInfo blockInfo = bm.getBlockInfo(blockId);
    Assert.assertEquals(2, blockInfo.getLength());
    Assert.assertFalse(blockInfo.getLocations().isEmpty());
    final BlockWorker bw = mLocalAlluxioClusterResource.get().getWorker().getBlockWorker();
    Assert.assertTrue(bw.hasBlockMeta(blockId));
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
    mFileSystem.free(filePath);
    IntegrationTestUtils.waitForBlocksToBeFreed(bw, blockId);
    status = mFileSystem.getStatus(filePath);
    // Verify block metadata in master is still present after block freed.
    Assert.assertEquals(1, status.getBlockIds().size());
    blockInfo = bm.getBlockInfo(status.getBlockIds().get(0));
    Assert.assertEquals(2, blockInfo.getLength());
    // Verify the block has been removed from all workers.
    Assert.assertTrue(blockInfo.getLocations().isEmpty());
    Assert.assertFalse(bw.hasBlockMeta(blockId));
    // Verify the removed block is added to LostBlocks list.
    Assert.assertTrue(bm.getLostBlocks().contains(blockInfo.getBlockId()));
    mFileSystem.delete(filePath);
    try {
        // File is immediately gone after delete.
        mFileSystem.getStatus(filePath);
        Assert.fail(String.format("Expected file %s being deleted but it was not.", filePath));
    } catch (FileDoesNotExistException e) {
    // expected
    }
    // Execute the lost files detection.
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_LOST_FILES_DETECTION);
    // Verify the blocks are not in mLostBlocks.
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
}

Example 4 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamProcessLocal.

@Test
public void getInStreamProcessLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.hasProcessLocalWorker()).thenReturn(true);
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf));
    assertEquals(local, stream.getAddress());
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 5 with BlockWorker

use of alluxio.worker.block.BlockWorker in project alluxio by Alluxio.

the class FreeAndDeleteIntegrationTest method freeAndDeleteIntegration.

@Test
public void freeAndDeleteIntegration() throws Exception {
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, mWriteBoth);
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    URIStatus status = mFileSystem.getStatus(filePath);
    assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    final Long blockId = status.getBlockIds().get(0);
    BlockMaster bm = mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess().getMaster(BlockMaster.class);
    BlockInfo blockInfo = bm.getBlockInfo(blockId);
    assertEquals(2, blockInfo.getLength());
    assertFalse(blockInfo.getLocations().isEmpty());
    final BlockWorker bw = mLocalAlluxioClusterResource.get().getWorkerProcess().getWorker(BlockWorker.class);
    assertTrue(bw.hasBlockMeta(blockId));
    assertEquals(0, bm.getLostBlocksCount());
    mFileSystem.free(filePath);
    CommonUtils.waitFor("file is freed", () -> {
        try {
            return 0 == mFileSystem.getStatus(filePath).getInAlluxioPercentage();
        } catch (Exception e) {
            return false;
        }
    }, WAIT_OPTIONS);
    status = mFileSystem.getStatus(filePath);
    // Verify block metadata in master is still present after block freed.
    assertEquals(1, status.getBlockIds().size());
    blockInfo = bm.getBlockInfo(status.getBlockIds().get(0));
    assertEquals(2, blockInfo.getLength());
    // Verify the block has been removed from all workers.
    assertTrue(blockInfo.getLocations().isEmpty());
    assertFalse(bw.hasBlockMeta(blockId));
    // Verify the removed block is added to LostBlocks list.
    assertTrue(bm.isBlockLost(blockInfo.getBlockId()));
    mFileSystem.delete(filePath);
    try {
        // File is immediately gone after delete.
        mFileSystem.getStatus(filePath);
        Assert.fail(String.format("Expected file %s being deleted but it was not.", filePath));
    } catch (FileDoesNotExistException e) {
    // expected
    }
    // Verify the blocks are not in mLostBlocks.
    CommonUtils.waitFor("block is removed from mLostBlocks", () -> {
        try {
            return 0 == bm.getLostBlocksCount();
        } catch (Exception e) {
            return false;
        }
    }, WAIT_OPTIONS);
}