Examples with FileOutStream alluxio.client.file.FileOutStream used on opensource projects

Example 11 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class FreeAndDeleteIntegrationTest method freeAndDeleteIntegration.

@Test
public void freeAndDeleteIntegration() throws Exception {
    HeartbeatScheduler.await(HeartbeatContext.WORKER_BLOCK_SYNC, 5, TimeUnit.SECONDS);
    HeartbeatScheduler.await(HeartbeatContext.MASTER_LOST_FILES_DETECTION, 5, TimeUnit.SECONDS);
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, mWriteBoth);
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    final Long blockId = status.getBlockIds().get(0);
    BlockMaster bm = mLocalAlluxioClusterResource.get().getMaster().getInternalMaster().getBlockMaster();
    BlockInfo blockInfo = bm.getBlockInfo(blockId);
    Assert.assertEquals(2, blockInfo.getLength());
    Assert.assertFalse(blockInfo.getLocations().isEmpty());
    final BlockWorker bw = mLocalAlluxioClusterResource.get().getWorker().getBlockWorker();
    Assert.assertTrue(bw.hasBlockMeta(blockId));
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
    mFileSystem.free(filePath);
    IntegrationTestUtils.waitForBlocksToBeFreed(bw, blockId);
    status = mFileSystem.getStatus(filePath);
    // Verify block metadata in master is still present after block freed.
    Assert.assertEquals(1, status.getBlockIds().size());
    blockInfo = bm.getBlockInfo(status.getBlockIds().get(0));
    Assert.assertEquals(2, blockInfo.getLength());
    // Verify the block has been removed from all workers.
    Assert.assertTrue(blockInfo.getLocations().isEmpty());
    Assert.assertFalse(bw.hasBlockMeta(blockId));
    // Verify the removed block is added to LostBlocks list.
    Assert.assertTrue(bm.getLostBlocks().contains(blockInfo.getBlockId()));
    mFileSystem.delete(filePath);
    try {
        // File is immediately gone after delete.
        mFileSystem.getStatus(filePath);
        Assert.fail(String.format("Expected file %s being deleted but it was not.", filePath));
    } catch (FileDoesNotExistException e) {
    // expected
    }
    // Execute the lost files detection.
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_LOST_FILES_DETECTION);
    // Verify the blocks are not in mLostBlocks.
    Assert.assertTrue(bm.getLostBlocks().isEmpty());
}

Example 12 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class CpCommand method copyFile.

/**
   * Copies a file in the Alluxio filesystem.
   *
   * @param srcPath the source {@link AlluxioURI} (has to be a file)
   * @param dstPath the destination path in the Alluxio filesystem
   * @throws AlluxioException when Alluxio exception occurs
   * @throws IOException when non-Alluxio exception occurs
   */
private void copyFile(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException {
    try (Closer closer = Closer.create()) {
        OpenFileOptions openFileOptions = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE);
        FileInStream is = closer.register(mFileSystem.openFile(srcPath, openFileOptions));
        CreateFileOptions createFileOptions = CreateFileOptions.defaults();
        FileOutStream os = closer.register(mFileSystem.createFile(dstPath, createFileOptions));
        IOUtils.copy(is, os);
        System.out.println("Copied " + srcPath + " to " + dstPath);
    }
}

Example 13 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class AbstractFileOutStreamIntegrationTest method writeTwoIncreasingByteArraysToFile.

/**
   * Helper to write an Alluxio file with one increasing byte array, but using two separate
   * {@link FileOutStream#write(byte[], int, int)} invocations.
   *
   * @param filePath path of the tmp file
   * @param fileLen length of the file
   * @param op options to create file
   */
protected void writeTwoIncreasingByteArraysToFile(AlluxioURI filePath, int fileLen, CreateFileOptions op) throws Exception {
    try (FileOutStream os = mFileSystem.createFile(filePath, op)) {
        int len1 = fileLen / 2;
        int len2 = fileLen - len1;
        os.write(BufferUtils.getIncreasingByteArray(0, len1), 0, len1);
        os.write(BufferUtils.getIncreasingByteArray(len1, len2), 0, len2);
    }
}

Example 14 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class PersistPermissionIntegrationTest method asyncPersistPermission.

@Test
public void asyncPersistPermission() throws Exception {
    // Skip non-local and non-HDFS UFSs.
    Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs) || UnderFileSystemUtils.isHdfs(mUfs));
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, CreateFileOptions.defaults().setWriteType(WriteType.ASYNC_THROUGH));
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    CommonUtils.sleepMs(1);
    // check the file is completed but not persisted
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.TO_BE_PERSISTED.toString(), status.getPersistenceState());
    Assert.assertTrue(status.isCompleted());
    short fileMode = (short) status.getMode();
    short parentMode = (short) mFileSystem.getStatus(filePath.getParent()).getMode();
    IntegrationTestUtils.waitForPersist(mLocalAlluxioClusterResource, filePath);
    status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    // Check the permission of the created file and parent dir are in-sync between Alluxio and UFS.
    Assert.assertEquals(fileMode, mUfs.getMode(PathUtils.concatPath(mUfsRoot, filePath)));
    Assert.assertEquals(parentMode, mUfs.getMode(PathUtils.concatPath(mUfsRoot, filePath.getParent())));
}

Example 15 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class PersistPermissionIntegrationTest method syncPersistPermission.

@Test
public void syncPersistPermission() throws Exception {
    // Skip non-local and non-HDFS UFSs.
    Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs) || UnderFileSystemUtils.isHdfs(mUfs));
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os = mFileSystem.createFile(filePath, CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH));
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    // Check the file is persisted
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    Assert.assertTrue(status.isCompleted());
    short fileMode = (short) status.getMode();
    short parentMode = (short) mFileSystem.getStatus(filePath.getParent()).getMode();
    // Check the permission of the created file and parent dir are in-sync between Alluxio and UFS.
    Assert.assertEquals(fileMode, mUfs.getMode(PathUtils.concatPath(mUfsRoot, filePath)));
    Assert.assertEquals(parentMode, mUfs.getMode(PathUtils.concatPath(mUfsRoot, filePath.getParent())));
}