Examples with FileBlockInfo alluxio.wire.FileBlockInfo used on opensource projects

Example 26 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class AbstractFileSystem method getFileBlockLocations.

@Nullable
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException {
    LOG.debug("getFileBlockLocations({}, {}, {})", (file == null) ? null : file.getPath().getName(), start, len);
    if (file == null) {
        LOG.debug("getFileBlockLocations({}, {}, {}) returned null", null, start, len);
        return null;
    }
    if (mStatistics != null) {
        mStatistics.incrementReadOps(1);
    }
    List<BlockLocation> blockLocations = new ArrayList<>();
    AlluxioURI path = getAlluxioPath(file.getPath());
    try {
        List<BlockLocationInfo> locations = mFileSystem.getBlockLocations(path);
        locations.forEach(location -> {
            FileBlockInfo info = location.getBlockInfo();
            List<WorkerNetAddress> workers = location.getLocations();
            long offset = location.getBlockInfo().getOffset();
            long end = offset + info.getBlockInfo().getLength();
            if (end >= start && offset <= start + len) {
                List<HostAndPort> addresses = workers.stream().map(worker -> HostAndPort.fromParts(worker.getHost(), worker.getDataPort())).collect(toList());
                String[] names = addresses.stream().map(HostAndPort::toString).toArray(String[]::new);
                String[] hosts = addresses.stream().map(HostAndPort::getHost).toArray(String[]::new);
                blockLocations.add(new BlockLocation(names, hosts, offset, info.getBlockInfo().getLength()));
            }
        });
        BlockLocation[] ret = blockLocations.toArray(new BlockLocation[blockLocations.size()]);
        if (LOG.isDebugEnabled()) {
            LOG.debug("getFileBlockLocations({}, {}, {}) returned {}", file.getPath().getName(), start, len, Arrays.toString(ret));
        }
        return ret;
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
}

Example 27 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class AbstractFileSystemTest method verifyBlockLocations.

void verifyBlockLocations(List<WorkerNetAddress> blockWorkers, List<String> ufsLocations, List<WorkerNetAddress> allWorkers, List<WorkerNetAddress> expectedWorkers) throws Exception {
    FileBlockInfo blockInfo = new FileBlockInfo().setBlockInfo(new BlockInfo().setLocations(blockWorkers.stream().map(addr -> new alluxio.wire.BlockLocation().setWorkerAddress(addr)).collect(toList()))).setUfsLocations(ufsLocations);
    FileInfo fileInfo = new FileInfo().setLastModificationTimeMs(111L).setLastAccessTimeMs(123L).setFolder(false).setOwner("user1").setGroup("group1").setMode(00755).setFileBlockInfos(Arrays.asList(blockInfo));
    Path path = new Path("/dir/file");
    AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
    AlluxioBlockStore blockStore = mock(AlluxioBlockStore.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(any(FileSystemContext.class))).thenReturn(blockStore);
    FileSystemContext fsContext = mock(FileSystemContext.class);
    when(fsContext.getClientContext()).thenReturn(ClientContext.create(mConfiguration));
    when(fsContext.getClusterConf()).thenReturn(mConfiguration);
    when(fsContext.getPathConf(any(AlluxioURI.class))).thenReturn(mConfiguration);
    alluxio.client.file.FileSystem fs = alluxio.client.file.FileSystem.Factory.create(fsContext);
    alluxio.client.file.FileSystem spyFs = spy(fs);
    doReturn(new URIStatus(fileInfo)).when(spyFs).getStatus(uri);
    List<BlockWorkerInfo> eligibleWorkerInfos = allWorkers.stream().map(worker -> new BlockWorkerInfo(worker, 0, 0)).collect(toList());
    when(fsContext.getCachedWorkers()).thenReturn(eligibleWorkerInfos);
    List<HostAndPort> expectedWorkerNames = expectedWorkers.stream().map(addr -> HostAndPort.fromParts(addr.getHost(), addr.getDataPort())).collect(toList());
    FileSystem alluxioHadoopFs = new FileSystem(spyFs);
    FileStatus file = new FileStatus(0, false, 0, 0, 0, 0, null, null, null, path);
    long start = 0;
    long len = 100;
    BlockLocation[] locations = alluxioHadoopFs.getFileBlockLocations(file, start, len);
    assertEquals(1, locations.length);
    Collections.sort(expectedWorkerNames, (x, y) -> x.toString().compareTo(y.toString()));
    String[] actualNames = locations[0].getNames();
    String[] actualHosts = locations[0].getHosts();
    Arrays.sort(actualNames);
    Arrays.sort(actualHosts);
    assertArrayEquals(expectedWorkerNames.stream().map(HostAndPort::toString).toArray(), actualNames);
    assertArrayEquals(expectedWorkerNames.stream().map(HostAndPort::getHost).toArray(), actualHosts);
    alluxioHadoopFs.close();
}

Example 28 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method scheduleAsyncPersist.

@Test
public void scheduleAsyncPersist() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    assertEquals(1, persistFiles.size());
    assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds());
}

Example 29 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method persistenceFileAfterDeletion.

/**
 * Tests persistence after deletion of files.
 */
@Test
public void persistenceFileAfterDeletion() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenThrow(new FileDoesNotExistException("no file"));
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    assertEquals(0, persistFiles.size());
}

Example 30 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class ReplicateDefinitionTest method before.

@Before
public void before() throws Exception {
    mMockFileSystemContext = PowerMockito.mock(FileSystemContext.class);
    when(mMockFileSystemContext.getClientContext()).thenReturn(ClientContext.create(ServerConfiguration.global()));
    when(mMockFileSystemContext.getClusterConf()).thenReturn(ServerConfiguration.global());
    mMockBlockStore = PowerMockito.mock(AlluxioBlockStore.class);
    mMockFileSystem = mock(FileSystem.class);
    mMockUfsManager = mock(UfsManager.class);
    mMockJobServerContext = new JobServerContext(mMockFileSystem, mMockFileSystemContext, mMockUfsManager);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    when(AlluxioBlockStore.create(mMockFileSystemContext)).thenReturn(mMockBlockStore);
    mTestBlockInfo = new BlockInfo().setBlockId(TEST_BLOCK_ID).setLength(TEST_BLOCK_SIZE);
    when(mMockBlockStore.getInfo(TEST_BLOCK_ID)).thenReturn(mTestBlockInfo);
    mTestStatus = new URIStatus(new FileInfo().setPath(TEST_PATH).setBlockIds(Lists.newArrayList(TEST_BLOCK_ID)).setPersisted(true).setFileBlockInfos(Lists.newArrayList(new FileBlockInfo().setBlockInfo(mTestBlockInfo))));
}