Examples with URIStatus alluxio.client.file.URIStatus used on opensource projects

Example 1 with URIStatus

use of alluxio.client.file.URIStatus in project zeppelin by apache.

the class AlluxioInterpreterTest method lsRecursiveTest.

@Test
public void lsRecursiveTest() throws IOException, AlluxioException {
    URIStatus[] files = new URIStatus[4];
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10, 10);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20, 20);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WriteType.THROUGH, 30, 30);
    files[0] = fs.getStatus(new AlluxioURI("/testRoot/testFileA"));
    files[1] = fs.getStatus(new AlluxioURI("/testRoot/testDir"));
    files[2] = fs.getStatus(new AlluxioURI("/testRoot/testDir/testFileB"));
    files[3] = fs.getStatus(new AlluxioURI("/testRoot/testFileC"));
    InterpreterResult output = alluxioInterpreter.interpret("ls -R /testRoot", null);
    String expected = "";
    String format = "%-10s%-25s%-15s%-5s\n";
    expected += String.format(format, FormatUtils.getSizeFromBytes(10), CommandUtils.convertMsToDate(files[0].getCreationTimeMs()), "In Memory", "/testRoot/testFileA");
    expected += String.format(format, FormatUtils.getSizeFromBytes(0), CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir");
    expected += String.format(format, FormatUtils.getSizeFromBytes(20), CommandUtils.convertMsToDate(files[2].getCreationTimeMs()), "In Memory", "/testRoot/testDir/testFileB");
    expected += String.format(format, FormatUtils.getSizeFromBytes(30), CommandUtils.convertMsToDate(files[3].getCreationTimeMs()), "Not In Memory", "/testRoot/testFileC");
    expected += "\n";
    Assert.assertEquals(expected, output.message().get(0).getData());
}

Example 2 with URIStatus

use of alluxio.client.file.URIStatus in project alluxio by Alluxio.

the class FileDataManagerTest method errorHandling.

/**
   * Tests that the correct error message is provided when persisting a file fails.
   */
@Test
public void errorHandling() throws Exception {
    long fileId = 1;
    List<Long> blockIds = Lists.newArrayList(1L, 2L);
    FileInfo fileInfo = new FileInfo();
    fileInfo.setPath("test");
    Mockito.when(mBlockWorker.getFileInfo(fileId)).thenReturn(fileInfo);
    for (long blockId : blockIds) {
        Mockito.when(mBlockWorker.lockBlock(Sessions.CHECKPOINT_SESSION_ID, blockId)).thenReturn(blockId);
        Mockito.doThrow(new InvalidWorkerStateException("invalid worker")).when(mBlockWorker).readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, blockId);
    }
    String ufsRoot = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
    Mockito.when(mUfs.isDirectory(ufsRoot)).thenReturn(true);
    OutputStream outputStream = Mockito.mock(OutputStream.class);
    // mock BufferUtils
    PowerMockito.mockStatic(BufferUtils.class);
    String dstPath = PathUtils.concatPath(ufsRoot, fileInfo.getPath());
    fileInfo.setUfsPath(dstPath);
    Mockito.when(mUfs.create(dstPath)).thenReturn(outputStream);
    Mockito.when(mUfs.create(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(outputStream);
    Mockito.when(mMockFileSystem.getStatus(Mockito.any(AlluxioURI.class))).thenReturn(new URIStatus(fileInfo));
    mManager.lockBlocks(fileId, blockIds);
    try {
        mManager.persistFile(fileId, blockIds);
        Assert.fail("the persist should fail");
    } catch (IOException e) {
        assertEquals("the blocks of file1 are failed to persist\n" + "alluxio.exception.InvalidWorkerStateException: invalid worker\n", e.getMessage());
        // verify the locks are all unlocked
        Mockito.verify(mBlockWorker).unlockBlock(1L);
        Mockito.verify(mBlockWorker).unlockBlock(2L);
    }
}

Example 3 with URIStatus

use of alluxio.client.file.URIStatus in project alluxio by Alluxio.

the class CapacityUsageIntegrationTest method deleteDuringEviction.

private void deleteDuringEviction(int i) throws IOException, AlluxioException {
    final AlluxioURI fileName1 = new AlluxioURI("/file" + i + "_1");
    final AlluxioURI fileName2 = new AlluxioURI("/file" + i + "_2");
    createAndWriteFile(fileName1, WriteType.CACHE_THROUGH, MEM_CAPACITY_BYTES);
    URIStatus fileStatus1 = mFileSystem.getStatus(fileName1);
    Assert.assertTrue(fileStatus1.getInMemoryPercentage() == 100);
    // Deleting file1, command will be sent by master to worker asynchronously
    mFileSystem.delete(fileName1);
    // Meanwhile creating file2. If creation arrives earlier than deletion, it will evict file1
    createAndWriteFile(fileName2, WriteType.CACHE_THROUGH, MEM_CAPACITY_BYTES / 4);
    URIStatus fileStatus2 = mFileSystem.getStatus(fileName2);
    Assert.assertTrue(fileStatus2.getInMemoryPercentage() == 100);
    mFileSystem.delete(fileName2);
}

Example 4 with URIStatus

use of alluxio.client.file.URIStatus in project alluxio by Alluxio.

the class SpecificTierWriteIntegrationTest method deleteAllFiles.

private void deleteAllFiles() throws Exception {
    List<URIStatus> files = mFileSystem.listStatus(new AlluxioURI("/"));
    for (URIStatus file : files) {
        mFileSystem.delete(new AlluxioURI(file.getPath()), DeleteOptions.defaults().setRecursive(true));
    }
    // Trigger a worker heartbeat to delete the blocks.
    HeartbeatScheduler.execute(HeartbeatContext.WORKER_BLOCK_SYNC);
    CommonUtils.waitFor("files to be deleted", new Function<Void, Boolean>() {

        @Override
        public Boolean apply(Void input) {
            try {
                // Trigger a worker heartbeat to report removed blocks.
                HeartbeatScheduler.execute(HeartbeatContext.WORKER_BLOCK_SYNC);
            } catch (InterruptedException e) {
            // ignore the exception
            }
            return mBlockMaster.getUsedBytes() == 0;
        }
    }, WaitForOptions.defaults().setTimeout(10 * Constants.SECOND_MS));
}

Example 5 with URIStatus

use of alluxio.client.file.URIStatus in project alluxio by Alluxio.

the class TieredStoreIntegrationTest method promoteBlock.

/**
   * Tests the promotion of a file.
   */
@Test
public void promoteBlock() throws Exception {
    AlluxioURI uri1 = new AlluxioURI("/file1");
    AlluxioURI uri2 = new AlluxioURI("/file2");
    AlluxioURI uri3 = new AlluxioURI("/file3");
    FileSystemTestUtils.createByteFile(mFileSystem, uri1, WriteType.CACHE_THROUGH, MEM_CAPACITY_BYTES / 6);
    FileSystemTestUtils.createByteFile(mFileSystem, uri2, WriteType.CACHE_THROUGH, MEM_CAPACITY_BYTES / 2);
    FileSystemTestUtils.createByteFile(mFileSystem, uri3, WriteType.CACHE_THROUGH, MEM_CAPACITY_BYTES / 2);
    HeartbeatScheduler.execute(HeartbeatContext.WORKER_BLOCK_SYNC);
    AlluxioURI toPromote;
    int toPromoteLen;
    URIStatus file1Info = mFileSystem.getStatus(uri1);
    URIStatus file2Info = mFileSystem.getStatus(uri2);
    URIStatus file3Info = mFileSystem.getStatus(uri3);
    // any assumptions on the eviction policy
    if (file1Info.getInMemoryPercentage() < 100) {
        toPromote = uri1;
        toPromoteLen = (int) file1Info.getLength();
        Assert.assertEquals(100, file2Info.getInMemoryPercentage());
        Assert.assertEquals(100, file3Info.getInMemoryPercentage());
    } else if (file2Info.getInMemoryPercentage() < 100) {
        toPromote = uri2;
        toPromoteLen = (int) file2Info.getLength();
        Assert.assertEquals(100, file1Info.getInMemoryPercentage());
        Assert.assertEquals(100, file3Info.getInMemoryPercentage());
    } else {
        toPromote = uri3;
        toPromoteLen = (int) file3Info.getLength();
        Assert.assertEquals(100, file1Info.getInMemoryPercentage());
        Assert.assertEquals(100, file2Info.getInMemoryPercentage());
    }
    FileInStream is = mFileSystem.openFile(toPromote, OpenFileOptions.defaults().setReadType(ReadType.CACHE_PROMOTE));
    byte[] buf = new byte[toPromoteLen];
    int len = is.read(buf);
    is.close();
    HeartbeatScheduler.schedule(HeartbeatContext.WORKER_BLOCK_SYNC);
    HeartbeatScheduler.await(HeartbeatContext.WORKER_BLOCK_SYNC, 10, TimeUnit.SECONDS);
    Assert.assertEquals(toPromoteLen, len);
    Assert.assertEquals(100, mFileSystem.getStatus(toPromote).getInMemoryPercentage());
}