Examples with FileBlockInfo alluxio.wire.FileBlockInfo used on opensource projects

Example 31 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileInStreamIntegrationTest method asyncCacheFirstBlock.

@Test(timeout = 10000)
public void asyncCacheFirstBlock() throws Exception {
    String filename = mTestPath + "/file_" + MAX_LEN + "_" + mWriteUnderStore.hashCode();
    AlluxioURI uri = new AlluxioURI(filename);
    for (ReadType readType : ReadType.values()) {
        mFileSystem.free(uri);
        CommonUtils.waitFor("No in-Alluxio data left from previous iteration.", () -> {
            try {
                URIStatus st = mFileSystem.getStatus(uri);
                return st.getInAlluxioPercentage() == 0;
            } catch (Exception e) {
                return false;
            }
        });
        FileInStream is = mFileSystem.openFile(uri, OpenFilePOptions.newBuilder().setReadType(readType.toProto()).build());
        is.read();
        URIStatus status = mFileSystem.getStatus(uri);
        Assert.assertEquals(0, status.getInAlluxioPercentage());
        is.close();
        if (readType.isCache()) {
            CommonUtils.waitFor("First block to be cached.", () -> {
                try {
                    URIStatus st = mFileSystem.getStatus(uri);
                    boolean achieved = true;
                    // Expect only first block to be cached, other blocks should be empty in Alluxio
                    for (int i = 0; i < st.getFileBlockInfos().size(); i++) {
                        FileBlockInfo info = st.getFileBlockInfos().get(i);
                        if (i == 0) {
                            achieved = achieved && !info.getBlockInfo().getLocations().isEmpty();
                        } else {
                            achieved = achieved && info.getBlockInfo().getLocations().isEmpty();
                        }
                    }
                    return achieved;
                } catch (Exception e) {
                    return false;
                }
            });
        } else {
            Thread.sleep(1000);
            status = mFileSystem.getStatus(uri);
            Assert.assertEquals(0, status.getInAlluxioPercentage());
        }
    }
}

Example 32 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileInStreamIntegrationTest method asyncCacheFirstBlockPRead.

@Test(timeout = 10000)
public void asyncCacheFirstBlockPRead() throws Exception {
    String filename = mTestPath + "/file_" + MAX_LEN + "_" + mWriteUnderStore.hashCode();
    AlluxioURI uri = new AlluxioURI(filename);
    for (ReadType readType : ReadType.values()) {
        mFileSystem.free(uri);
        CommonUtils.waitFor("No in-Alluxio data left from previous iteration.", () -> {
            try {
                URIStatus st = mFileSystem.getStatus(uri);
                return st.getInAlluxioPercentage() == 0;
            } catch (Exception e) {
                return false;
            }
        });
        FileInStream is = mFileSystem.openFile(uri, OpenFilePOptions.newBuilder().setReadType(readType.toProto()).build());
        // Positioned reads trigger async caching after reading and do not need to wait for a close
        // or a block boundary to be crossed.
        URIStatus status = mFileSystem.getStatus(uri);
        Assert.assertEquals(0, status.getInAlluxioPercentage());
        is.positionedRead(BLOCK_SIZE / 2, new byte[1], 0, 1);
        if (readType.isCache()) {
            CommonUtils.waitFor("First block to be cached.", () -> {
                try {
                    URIStatus st = mFileSystem.getStatus(uri);
                    boolean achieved = true;
                    // Expect only first block to be cached, other blocks should be empty in Alluxio
                    for (int i = 0; i < st.getFileBlockInfos().size(); i++) {
                        FileBlockInfo info = st.getFileBlockInfos().get(i);
                        if (i == 0) {
                            achieved = achieved && !info.getBlockInfo().getLocations().isEmpty();
                        } else {
                            achieved = achieved && info.getBlockInfo().getLocations().isEmpty();
                        }
                    }
                    return achieved;
                } catch (Exception e) {
                    return false;
                }
            });
        } else {
            Thread.sleep(1000);
            status = mFileSystem.getStatus(uri);
            Assert.assertEquals(0, status.getInAlluxioPercentage());
        }
        is.close();
    }
}

Example 33 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class FileOutStreamAsyncWriteIntegrationTest method testLostAsyncBlocks.

/**
 * Test eviction against a file that is slow to persist in Alluxio. The test cluster should be
 * configured with a high initial wait time, or -1
 *
 * This test performs the following actions:
 * - creates a file with ASYNC_THROUGH which fills the entire capacity of a worker
 * - Tries to create another 1-byte file on top and expects it to fail
 * - persists the file to the UFS
 * - Tries to create 1-byte file again and it should succeed this time
 */
private void testLostAsyncBlocks() throws Exception {
    long cap = FormatUtils.parseSpaceSize(TINY_WORKER_MEM);
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient();
    // Create a large-ish file in the UFS (relative to memory size)
    String p1 = "/test";
    FileSystemTestUtils.createByteFile(fs, p1, WritePType.ASYNC_THROUGH, (int) cap);
    URIStatus fstat = fs.listStatus(new AlluxioURI(p1)).get(0);
    int lostBlocks = fstat.getFileBlockInfos().stream().map(FileBlockInfo::getBlockInfo).filter(blk -> blk.getLocations().size() <= 0).mapToInt(blk -> 1).sum();
    assertEquals(cap, getClusterCapacity());
    assertEquals(cap, getUsedWorkerSpace());
    assertEquals(100, fstat.getInAlluxioPercentage());
    assertEquals(0, lostBlocks);
    // Try to create 1-byte file on top. Expect to fail.
    try {
        FileSystemTestUtils.createByteFile(fs, "/byte-file1", WritePType.MUST_CACHE, 1);
        assertTrue("Shouldn't reach here.", false);
    } catch (Exception e) {
    // expected.
    }
    FileSystemUtils.persistAndWait(fs, new AlluxioURI(p1), 0);
    fstat = fs.listStatus(new AlluxioURI(p1)).get(0);
    assertTrue(fstat.isPersisted());
    assertEquals(0, mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class).getPinIdList().size());
    // Try to create 1-byte file on top. Expect to succeed.
    FileSystemTestUtils.createByteFile(fs, "/byte-file2", WritePType.MUST_CACHE, 1);
}

Example 34 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class GrpcUtils method toProto.

/**
 * Converts a wire type to a proto type.
 *
 * @param fileInfo the wire representation of a file information
 * @return proto representation of the file information
 */
public static alluxio.grpc.FileInfo toProto(FileInfo fileInfo) {
    List<alluxio.grpc.FileBlockInfo> fileBlockInfos = new ArrayList<>();
    for (FileBlockInfo fileBlockInfo : fileInfo.getFileBlockInfos()) {
        fileBlockInfos.add(toProto(fileBlockInfo));
    }
    alluxio.grpc.FileInfo.Builder builder = alluxio.grpc.FileInfo.newBuilder().setFileId(fileInfo.getFileId()).setName(fileInfo.getName()).setPath(fileInfo.getPath()).setUfsPath(fileInfo.getUfsPath()).setLength(fileInfo.getLength()).setBlockSizeBytes(fileInfo.getBlockSizeBytes()).setCreationTimeMs(fileInfo.getCreationTimeMs()).setCompleted(fileInfo.isCompleted()).setFolder(fileInfo.isFolder()).setPinned(fileInfo.isPinned()).setCacheable(fileInfo.isCacheable()).setPersisted(fileInfo.isPersisted()).addAllBlockIds(fileInfo.getBlockIds()).setLastModificationTimeMs(fileInfo.getLastModificationTimeMs()).setTtl(fileInfo.getTtl()).setLastAccessTimeMs(fileInfo.getLastAccessTimeMs()).setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()).setMode(fileInfo.getMode()).setPersistenceState(fileInfo.getPersistenceState()).setMountPoint(fileInfo.isMountPoint()).addAllFileBlockInfos(fileBlockInfos).setTtlAction(fileInfo.getTtlAction()).setMountId(fileInfo.getMountId()).setInAlluxioPercentage(fileInfo.getInAlluxioPercentage()).setInMemoryPercentage(fileInfo.getInMemoryPercentage()).setUfsFingerprint(fileInfo.getUfsFingerprint()).setReplicationMax(fileInfo.getReplicationMax()).setReplicationMin(fileInfo.getReplicationMin());
    if (!fileInfo.getAcl().equals(AccessControlList.EMPTY_ACL)) {
        builder.setAcl(toProto(fileInfo.getAcl()));
    }
    if (!fileInfo.getDefaultAcl().equals(DefaultAccessControlList.EMPTY_DEFAULT_ACL)) {
        builder.setDefaultAcl(toProto(fileInfo.getDefaultAcl()));
    }
    if (fileInfo.getXAttr() != null) {
        for (Map.Entry<String, byte[]> entry : fileInfo.getXAttr().entrySet()) {
            builder.putXattr(entry.getKey(), ByteString.copyFrom(entry.getValue()));
        }
    }
    if (!fileInfo.getMediumTypes().isEmpty()) {
        builder.addAllMediumType(fileInfo.getMediumTypes());
    }
    return builder.build();
}

Example 35 with FileBlockInfo

use of alluxio.wire.FileBlockInfo in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method before.

/**
 * Sets up the context and streams before a test runs.
 */
@Before
public void before() throws Exception {
    mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(mFileSize);
    ClientTestUtils.setSmallBufferSizes(mConf);
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN, "1ms");
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MAX, "5ms");
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_MAX_DURATION, "1s");
    BlockWorkerClient client = mock(BlockWorkerClient.class);
    doNothing().when(client).cache(any());
    mContext = mock(FileSystemContext.class);
    when(mContext.getClientContext()).thenReturn(ClientContext.create(mConf));
    when(mContext.getClusterConf()).thenReturn(mConf);
    when(mContext.getPathConf(any(AlluxioURI.class))).thenReturn(mConf);
    when(mContext.getNodeLocalWorker()).thenReturn(new WorkerNetAddress());
    when(mContext.getCachedWorkers()).thenReturn(new ArrayList<>());
    when(mContext.acquireBlockWorkerClient(any())).thenReturn(new CloseableResource<BlockWorkerClient>(client) {

        @Override
        public void closeResource() {
        }
    });
    mBlockStore = mock(AlluxioBlockStore.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mContext)).thenReturn(mBlockStore);
    // Set up BufferedBlockInStreams and caching streams
    mInStreams = new ArrayList<>();
    List<Long> blockIds = new ArrayList<>();
    List<FileBlockInfo> fileBlockInfos = new ArrayList<>();
    for (int i = 0; i < mNumBlocks; i++) {
        blockIds.add((long) i);
        FileBlockInfo fbInfo = new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(i));
        fileBlockInfos.add(fbInfo);
        final byte[] input = BufferUtils.getIncreasingByteArray((int) (i * BLOCK_LENGTH), (int) getBlockLength(i));
        mInStreams.add(new TestBlockInStream(input, i, input.length, false, mBlockSource));
        when(mContext.getCachedWorkers()).thenReturn(Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress(), 0, 0)));
        when(mBlockStore.getInStream(eq((long) i), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
            long blockId = (Long) invocation.getArguments()[0];
            return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
        });
        when(mBlockStore.getInStream(eq(new BlockInfo().setBlockId(i)), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
            long blockId = ((BlockInfo) invocation.getArguments()[0]).getBlockId();
            return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
        });
    }
    mInfo.setBlockIds(blockIds);
    mInfo.setFileBlockInfos(fileBlockInfos).setReplicationMax(1);
    mStatus = new URIStatus(mInfo);
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, mConf), mContext);
}