Examples with FileBlockInfo alluxio.wire.FileBlockInfo used on opensource projects

Example 16 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileInStreamIntegrationTest method asyncCacheAfterSeek.

@Test(timeout = 10000)
public void asyncCacheAfterSeek() throws Exception {
    String filename = mTestPath + "/file_" + MAX_LEN + "_" + mWriteUnderStore.hashCode();
    AlluxioURI uri = new AlluxioURI(filename);
    for (ReadType readType : ReadType.values()) {
        mFileSystem.free(uri);
        CommonUtils.waitFor("No in-Alluxio data left from previous iteration.", () -> {
            try {
                URIStatus st = mFileSystem.getStatus(uri);
                return st.getInAlluxioPercentage() == 0;
            } catch (Exception e) {
                return false;
            }
        });
        FileInStream is = mFileSystem.openFile(uri, OpenFilePOptions.newBuilder().setReadType(readType.toProto()).build());
        URIStatus status = mFileSystem.getStatus(uri);
        is.seek(status.getBlockSizeBytes() + 1);
        is.read();
        status = mFileSystem.getStatus(uri);
        Assert.assertEquals(0, status.getInAlluxioPercentage());
        is.close();
        if (readType.isCache()) {
            CommonUtils.waitFor("Second block to be cached.", () -> {
                try {
                    URIStatus st = mFileSystem.getStatus(uri);
                    boolean achieved = true;
                    // Expect only second block to be cached, other blocks should be empty in Alluxio
                    for (int i = 0; i < st.getFileBlockInfos().size(); i++) {
                        FileBlockInfo info = st.getFileBlockInfos().get(i);
                        if (i == 1) {
                            achieved = achieved && !info.getBlockInfo().getLocations().isEmpty();
                        } else {
                            achieved = achieved && info.getBlockInfo().getLocations().isEmpty();
                        }
                    }
                    return achieved;
                } catch (Exception e) {
                    return false;
                }
            });
        } else {
            Thread.sleep(1000);
            status = mFileSystem.getStatus(uri);
            Assert.assertEquals(0, status.getInAlluxioPercentage());
        }
    }
}

Example 17 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class MultiWorkerIntegrationTest method replicateFileBlocks.

private void replicateFileBlocks(AlluxioURI filePath) throws Exception {
    FileSystemContext fsContext = FileSystemContext.create(ServerConfiguration.global());
    AlluxioBlockStore store = AlluxioBlockStore.create(fsContext);
    URIStatus status = mResource.get().getClient().getStatus(filePath);
    List<FileBlockInfo> blocks = status.getFileBlockInfos();
    List<BlockWorkerInfo> workers = fsContext.getCachedWorkers();
    for (FileBlockInfo block : blocks) {
        BlockInfo blockInfo = block.getBlockInfo();
        WorkerNetAddress src = blockInfo.getLocations().get(0).getWorkerAddress();
        WorkerNetAddress dest = workers.stream().filter(candidate -> !candidate.getNetAddress().equals(src)).findFirst().get().getNetAddress();
        try (OutputStream outStream = store.getOutStream(blockInfo.getBlockId(), blockInfo.getLength(), dest, OutStreamOptions.defaults(fsContext.getClientContext()).setBlockSizeBytes(8 * Constants.MB).setWriteType(WriteType.MUST_CACHE))) {
            try (InputStream inStream = store.getInStream(blockInfo.getBlockId(), new InStreamOptions(status, ServerConfiguration.global()))) {
                ByteStreams.copy(inStream, outStream);
            }
        }
    }
}

Example 18 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method persistenceFileWithBlocksOnMultipleWorkers.

/**
 * Tests the persistence of file with block on multiple workers.
 */
@Test
public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location1 = new BlockLocation().setWorkerId(1);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1))));
    BlockLocation location2 = new BlockLocation().setWorkerId(2);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2))));
    long fileId = 2;
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    // no persist scheduled on any worker
    assertEquals(0, handler.pollFilesToPersist(1).size());
    assertEquals(0, handler.pollFilesToPersist(2).size());
}

Example 19 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class BlockWorkerDataReaderTest method readChunkUfs.

@Test
public void readChunkUfs() throws Exception {
    // Write an actual file to UFS
    String testFilePath = File.createTempFile("temp", null, new File(mRootUfs)).getAbsolutePath();
    byte[] buffer = BufferUtils.getIncreasingByteArray(CHUNK_SIZE * 5);
    BufferUtils.writeBufferToFile(testFilePath, buffer);
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE * 5);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setUfsPath(testFilePath).setBlockIds(Collections.singletonList(BLOCK_ID)).setLength(CHUNK_SIZE * 5).setBlockSizeBytes(CHUNK_SIZE).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, mConf);
    BlockWorkerDataReader.Factory factory = new BlockWorkerDataReader.Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options);
    int len = CHUNK_SIZE * 3 / 2;
    try (DataReader dataReader = factory.create(0, len)) {
        validateBuffer(dataReader.readChunk(), 0, CHUNK_SIZE);
        assertEquals(CHUNK_SIZE, dataReader.pos());
        validateBuffer(dataReader.readChunk(), CHUNK_SIZE, len - CHUNK_SIZE);
        assertEquals(len, dataReader.pos());
    }
}

Example 20 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class JobUtils method getWorkerWithMostBlocks.

/**
 * Returns whichever specified worker stores the most blocks from the block info list.
 *
 * @param workers a list of workers to consider
 * @param fileBlockInfos a list of file block information
 * @return a worker address storing the most blocks from the list
 */
public static BlockWorkerInfo getWorkerWithMostBlocks(List<BlockWorkerInfo> workers, List<FileBlockInfo> fileBlockInfos) {
    // Index workers by their addresses.
    IndexedSet<BlockWorkerInfo> addressIndexedWorkers = new IndexedSet<>(WORKER_ADDRESS_INDEX);
    addressIndexedWorkers.addAll(workers);
    // Use ConcurrentMap for putIfAbsent. A regular Map works in Java 8.
    ConcurrentMap<BlockWorkerInfo, Integer> blocksPerWorker = Maps.newConcurrentMap();
    int maxBlocks = 0;
    BlockWorkerInfo mostBlocksWorker = null;
    for (FileBlockInfo fileBlockInfo : fileBlockInfos) {
        for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
            BlockWorkerInfo worker = addressIndexedWorkers.getFirstByField(WORKER_ADDRESS_INDEX, location.getWorkerAddress());
            if (worker == null) {
                // We can only choose workers in the workers list.
                continue;
            }
            blocksPerWorker.putIfAbsent(worker, 0);
            int newBlockCount = blocksPerWorker.get(worker) + 1;
            blocksPerWorker.put(worker, newBlockCount);
            if (newBlockCount > maxBlocks) {
                maxBlocks = newBlockCount;
                mostBlocksWorker = worker;
            }
        }
    }
    return mostBlocksWorker;
}