Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 81 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileDataManager method prepareUfsFilePath.

/**
   * Prepares the destination file path of the given file id. Also creates the parent folder if it
   * does not exist.
   *
   * @param fileId the file id
   * @return the path for persistence
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   * @throws IOException if the folder creation fails
   */
private String prepareUfsFilePath(long fileId) throws AlluxioException, IOException {
    FileInfo fileInfo = mBlockWorker.getFileInfo(fileId);
    AlluxioURI alluxioPath = new AlluxioURI(fileInfo.getPath());
    FileSystem fs = FileSystem.Factory.get();
    URIStatus status = fs.getStatus(alluxioPath);
    String ufsPath = status.getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsPath);
    UnderFileSystemUtils.prepareFilePath(alluxioPath, ufsPath, fs, ufs);
    return ufsPath;
}

Example 82 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UnderFileSystemBlockReader method init.

/**
   * Initializes the reader. This is only called in the factory method.
   *
   * @param offset the position within the block to start the read
   * @throws BlockDoesNotExistException if the UFS block does not exist in the UFS block store
   * @throws IOException if an I/O related error occur
   */
private void init(long offset) throws BlockDoesNotExistException, IOException {
    UnderFileSystem ufs = UnderFileSystem.Factory.get(mBlockMeta.getUnderFileSystemPath());
    ufs.connectFromWorker(NetworkAddressUtils.getConnectHost(NetworkAddressUtils.ServiceType.WORKER_RPC));
    if (!ufs.isFile(mBlockMeta.getUnderFileSystemPath())) {
        throw new BlockDoesNotExistException(ExceptionMessage.UFS_PATH_DOES_NOT_EXIST.getMessage(mBlockMeta.getUnderFileSystemPath()));
    }
    updateUnderFileSystemInputStream(offset);
    updateBlockWriter(offset);
}

Example 83 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class S3UnderStorageCluster method shutdown.

@Override
public void shutdown() throws IOException {
    LOG.info("Shutting down S3 testing cluster, deleting bucket contents in: " + mS3Bucket);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(mS3Bucket);
    ufs.deleteDirectory(mS3Bucket, DeleteOptions.defaults().setRecursive(true));
}

Example 84 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UnderFileSystemUtilsTest method beforeClass.

@BeforeClass
public static void beforeClass() {
    // For each UFS type, create a pair, add the pair to the object stores if necessary
    sPairs = new ArrayList<>();
    sObjectStores = new ArrayList<>();
    // GCS
    UnderFileSystem gcs = Mockito.mock(GCSUnderFileSystem.class);
    Mockito.when(gcs.getUnderFSType()).thenCallRealMethod();
    sPairs.add(new UfsTypeCheckPair(Collections.singletonList(gcs), new UfsTypeCheckCallable() {

        @Override
        public boolean call(UnderFileSystem ufs) {
            return UnderFileSystemUtils.isGcs(ufs);
        }
    }));
    sObjectStores.add(gcs);
    // HDFS
    UnderFileSystem hdfs = Mockito.mock(HdfsUnderFileSystem.class);
    Mockito.when(hdfs.getUnderFSType()).thenCallRealMethod();
    sPairs.add(new UfsTypeCheckPair(Collections.singletonList(hdfs), new UfsTypeCheckCallable() {

        @Override
        public boolean call(UnderFileSystem ufs) {
            return UnderFileSystemUtils.isHdfs(ufs);
        }
    }));
    // Local
    UnderFileSystem local = Mockito.mock(LocalUnderFileSystem.class);
    Mockito.when(local.getUnderFSType()).thenCallRealMethod();
    sPairs.add(new UfsTypeCheckPair(Collections.singletonList(local), new UfsTypeCheckCallable() {

        @Override
        public boolean call(UnderFileSystem ufs) {
            return UnderFileSystemUtils.isLocal(ufs);
        }
    }));
    // OSS
    UnderFileSystem oss = Mockito.mock(OSSUnderFileSystem.class);
    Mockito.when(oss.getUnderFSType()).thenCallRealMethod();
    sPairs.add(new UfsTypeCheckPair(Collections.singletonList(oss), new UfsTypeCheckCallable() {

        @Override
        public boolean call(UnderFileSystem ufs) {
            return UnderFileSystemUtils.isOss(ufs);
        }
    }));
    sObjectStores.add(oss);
    // S3
    UnderFileSystem s3 = Mockito.mock(S3UnderFileSystem.class);
    Mockito.when(s3.getUnderFSType()).thenCallRealMethod();
    UnderFileSystem s3a = Mockito.mock(S3AUnderFileSystem.class);
    Mockito.when(s3a.getUnderFSType()).thenCallRealMethod();
    sPairs.add(new UfsTypeCheckPair(Arrays.asList(s3, s3a), new UfsTypeCheckCallable() {

        @Override
        public boolean call(UnderFileSystem ufs) {
            return UnderFileSystemUtils.isS3(ufs);
        }
    }));
    sObjectStores.add(s3);
    sObjectStores.add(s3a);
    // Swift
    UnderFileSystem swift = Mockito.mock(SwiftUnderFileSystem.class);
    Mockito.when(swift.getUnderFSType()).thenCallRealMethod();
    sPairs.add(new UfsTypeCheckPair(Collections.singletonList(swift), new UfsTypeCheckCallable() {

        @Override
        public boolean call(UnderFileSystem ufs) {
            return UnderFileSystemUtils.isSwift(ufs);
        }
    }));
    sObjectStores.add(swift);
}

Example 85 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class BlockServiceHandlerIntegrationTest method lockBlock.

// Tests that lock block returns the correct path
@Test
public void lockBlock() throws Exception {
    final int blockSize = (int) WORKER_CAPACITY_BYTES / 2;
    CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(blockSize).setWriteType(WriteType.MUST_CACHE);
    FileOutStream out = mFileSystem.createFile(new AlluxioURI("/testFile"), options);
    URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));
    final long blockId = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);
    out.write(BufferUtils.getIncreasingByteArray(blockSize));
    out.close();
    String localPath = mBlockWorkerServiceHandler.lockBlock(blockId, SESSION_ID, new LockBlockTOptions()).getBlockPath();
    // The local path should exist
    Assert.assertNotNull(localPath);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(localPath);
    byte[] data = new byte[blockSize];
    InputStream in = ufs.open(localPath);
    int bytesRead = in.read(data);
    // The data in the local file should equal the data we wrote earlier
    Assert.assertEquals(blockSize, bytesRead);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytesRead, data));
    mBlockWorkerServiceHandler.unlockBlock(blockId, SESSION_ID);
}