Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 16 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class CheckConsistencyIntegrationTest method incorrectFileSize.

/**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyOptions)} method
   * when a file is not the correct size.
   */
@Test
public void incorrectFileSize() throws Exception {
    String ufsFile = mFileSystem.getStatus(FILE).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsFile);
    ufs.deleteFile(ufsFile);
    OutputStream out = ufs.create(ufsFile);
    out.write(1);
    out.close();
    List<AlluxioURI> expected = Lists.newArrayList(FILE);
    Assert.assertEquals(expected, mFileSystemMaster.checkConsistency(new AlluxioURI("/"), CheckConsistencyOptions.defaults()));
}

Example 17 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileDataManager method fileExistsInUfs.

/**
   * Checks if the given file exists in the under storage system.
   *
   * @param fileId the file id
   * @return true if the file exists in under storage system, false otherwise
   * @throws IOException an I/O exception occurs
   */
private synchronized boolean fileExistsInUfs(long fileId) throws IOException {
    FileInfo fileInfo = mBlockWorker.getFileInfo(fileId);
    String dstPath = fileInfo.getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(dstPath);
    return ufs.isFile(dstPath);
}

Example 18 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileDataManager method persistFile.

/**
   * Persists the blocks of a file into the under file system.
   *
   * @param fileId the id of the file
   * @param blockIds the list of block ids
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   * @throws IOException if the file persistence fails
   */
public void persistFile(long fileId, List<Long> blockIds) throws AlluxioException, IOException {
    Map<Long, Long> blockIdToLockId;
    synchronized (mLock) {
        blockIdToLockId = mPersistingInProgressFiles.get(fileId);
        if (blockIdToLockId == null || !blockIdToLockId.keySet().equals(new HashSet<>(blockIds))) {
            throw new IOException("Not all the blocks of file " + fileId + " are locked");
        }
    }
    String dstPath = prepareUfsFilePath(fileId);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(dstPath);
    FileInfo fileInfo = mBlockWorker.getFileInfo(fileId);
    OutputStream outputStream = ufs.create(dstPath, CreateOptions.defaults().setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()).setMode(new Mode((short) fileInfo.getMode())));
    final WritableByteChannel outputChannel = Channels.newChannel(outputStream);
    List<Throwable> errors = new ArrayList<>();
    try {
        for (long blockId : blockIds) {
            long lockId = blockIdToLockId.get(blockId);
            if (Configuration.getBoolean(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT_ENABLED)) {
                BlockMeta blockMeta = mBlockWorker.getBlockMeta(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
                mPersistenceRateLimiter.acquire((int) blockMeta.getBlockSize());
            }
            // obtain block reader
            BlockReader reader = mBlockWorker.readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, lockId);
            // write content out
            ReadableByteChannel inputChannel = reader.getChannel();
            BufferUtils.fastCopy(inputChannel, outputChannel);
            reader.close();
        }
    } catch (BlockDoesNotExistException | InvalidWorkerStateException e) {
        errors.add(e);
    } finally {
        // make sure all the locks are released
        for (long lockId : blockIdToLockId.values()) {
            try {
                mBlockWorker.unlockBlock(lockId);
            } catch (BlockDoesNotExistException e) {
                errors.add(e);
            }
        }
        // Process any errors
        if (!errors.isEmpty()) {
            StringBuilder errorStr = new StringBuilder();
            errorStr.append("the blocks of file").append(fileId).append(" are failed to persist\n");
            for (Throwable e : errors) {
                errorStr.append(e).append('\n');
            }
            throw new IOException(errorStr.toString());
        }
    }
    outputStream.flush();
    outputChannel.close();
    outputStream.close();
    synchronized (mLock) {
        mPersistingInProgressFiles.remove(fileId);
        mPersistedFiles.add(fileId);
    }
}

Example 19 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class GCSUnderStorageCluster method shutdown.

@Override
public void shutdown() throws IOException {
    LOG.info("Shutting down GCS testing cluster, deleting bucket contents in: " + mGCSBucket);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(mGCSBucket);
    ufs.deleteDirectory(mGCSBucket, DeleteOptions.defaults().setRecursive(true));
}

Example 20 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class BlockServiceHandlerIntegrationTest method createBlockFile.

// Creates a block file and write an increasing byte array into it
private void createBlockFile(String filename, int len) throws IOException, InvalidPathException {
    UnderFileSystem ufs = UnderFileSystem.Factory.get(filename);
    ufs.mkdirs(PathUtils.getParent(filename));
    OutputStream out = ufs.create(filename);
    out.write(BufferUtils.getIncreasingByteArray(len), 0, len);
    out.close();
}