Examples with BlockInfo alluxio.wire.BlockInfo used on opensource projects

Example 16 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class DataServerIntegrationTest method read.

@Test
public void read() throws Exception {
    final int length = 10;
    FileSystemTestUtils.createByteFile(mFileSystem, "/file", WriteType.MUST_CACHE, length);
    BlockInfo block = getFirstBlockInfo(new AlluxioURI("/file"));
    DataServerMessage recvMsg = request(block);
    assertValid(recvMsg, length, block.getBlockId(), 0, length);
}

Example 17 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class FreeAndDeleteIntegrationTest method freeAndDeleteIntegration.

@Test
public void freeAndDeleteIntegration() throws Exception {
    HeartbeatScheduler.await(HeartbeatContext.WORKER_BLOCK_SYNC, 5, TimeUnit.SECONDS);
    HeartbeatScheduler.await(HeartbeatContext.MASTER_LOST_FILES_DETECTION, 5, TimeUnit.SECONDS);
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, mWriteBoth);
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    final Long blockId = status.getBlockIds().get(0);
    BlockMaster bm = mLocalAlluxioClusterResource.get().getMaster().getInternalMaster().getBlockMaster();
    BlockInfo blockInfo = bm.getBlockInfo(blockId);
    Assert.assertEquals(2, blockInfo.getLength());
    Assert.assertFalse(blockInfo.getLocations().isEmpty());
    final BlockWorker bw = mLocalAlluxioClusterResource.get().getWorker().getBlockWorker();
    Assert.assertTrue(bw.hasBlockMeta(blockId));
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
    mFileSystem.free(filePath);
    IntegrationTestUtils.waitForBlocksToBeFreed(bw, blockId);
    status = mFileSystem.getStatus(filePath);
    // Verify block metadata in master is still present after block freed.
    Assert.assertEquals(1, status.getBlockIds().size());
    blockInfo = bm.getBlockInfo(status.getBlockIds().get(0));
    Assert.assertEquals(2, blockInfo.getLength());
    // Verify the block has been removed from all workers.
    Assert.assertTrue(blockInfo.getLocations().isEmpty());
    Assert.assertFalse(bw.hasBlockMeta(blockId));
    // Verify the removed block is added to LostBlocks list.
    Assert.assertTrue(bm.getLostBlocks().contains(blockInfo.getBlockId()));
    mFileSystem.delete(filePath);
    try {
        // File is immediately gone after delete.
        mFileSystem.getStatus(filePath);
        Assert.fail(String.format("Expected file %s being deleted but it was not.", filePath));
    } catch (FileDoesNotExistException e) {
    // expected
    }
    // Execute the lost files detection.
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_LOST_FILES_DETECTION);
    // Verify the blocks are not in mLostBlocks.
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
}

Example 18 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class RemoteBlockInStreamIntegrationTest method readTest6.

/**
   * Tests {@link RemoteBlockInStream#read(byte[], int, int)}. Read from remote data server.
   */
@Test
public void readTest6() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        long blockId = mFileSystem.getStatus(uri).getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create().getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        RemoteBlockInStream is = RemoteBlockInStream.create(info.getBlockId(), info.getLength(), workerAddr, FileSystemContext.INSTANCE, InStreamOptions.defaults());
        byte[] ret = new byte[k / 2];
        int start = 0;
        while (start < k / 2) {
            int read = is.read(ret, 0, (k / 2) - start);
            Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
            start += read;
        }
        is.close();
        Assert.assertTrue(mFileSystem.getStatus(uri).getInMemoryPercentage() == 100);
    }
}

Example 19 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class RemoteBlockInStreamIntegrationTest method remoteReadLock.

/**
   * Tests remote read stream lock in {@link RemoteBlockInStream}.
   */
@Test
public void remoteReadLock() throws Exception {
    HeartbeatScheduler.await(HeartbeatContext.WORKER_BLOCK_SYNC, 10, TimeUnit.SECONDS);
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        HeartbeatScheduler.execute(HeartbeatContext.WORKER_BLOCK_SYNC);
        long blockId = mFileSystem.getStatus(uri).getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create().getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        RemoteBlockInStream is = RemoteBlockInStream.create(info.getBlockId(), info.getLength(), workerAddr, FileSystemContext.INSTANCE, InStreamOptions.defaults());
        Assert.assertEquals(0, is.read());
        mFileSystem.delete(uri);
        HeartbeatScheduler.execute(HeartbeatContext.WORKER_BLOCK_SYNC);
        // The file has been deleted.
        Assert.assertFalse(mFileSystem.exists(uri));
        // Look! We can still read the deleted file since we have a lock!
        byte[] ret = new byte[k / 2];
        Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
        is.close();
        Assert.assertFalse(mFileSystem.exists(uri));
        // Try to create an in stream again, and it should fail.
        RemoteBlockInStream is2 = null;
        try {
            is2 = RemoteBlockInStream.create(info.getBlockId(), info.getLength(), workerAddr, FileSystemContext.INSTANCE, InStreamOptions.defaults());
        } catch (IOException e) {
            Assert.assertTrue(e.getCause() instanceof BlockDoesNotExistException);
        } finally {
            if (is2 != null) {
                is2.close();
            }
        }
    }
}

Example 20 with BlockInfo

use of alluxio.wire.BlockInfo in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method scheduleAsyncPersist.

@Test
public void scheduleAsyncPersist() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    Mockito.when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    Mockito.when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    Mockito.when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    Mockito.when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    Assert.assertEquals(1, persistFiles.size());
    Assert.assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds());
}