Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 76 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class TestRunner method runTests.

/**
 * Runs combinations of tests of operation, read and write type.
 *
 * @return the number of failed tests
 */
private int runTests() throws Exception {
    mDirectory = PathUtils.concatPath(mDirectory, TEST_DIRECTORY_NAME);
    AlluxioURI testDir = new AlluxioURI(mDirectory);
    FileSystemContext fsContext = FileSystemContext.create(new InstancedConfiguration(ConfigurationUtils.defaults()));
    FileSystem fs = FileSystem.Factory.create(fsContext);
    if (fs.exists(testDir)) {
        fs.delete(testDir, DeletePOptions.newBuilder().setRecursive(true).setUnchecked(true).build());
    }
    int failed = 0;
    List<ReadType> readTypes = mReadType == null ? READ_TYPES : Lists.newArrayList(ReadType.valueOf(mReadType));
    List<WriteType> writeTypes = mWriteType == null ? WRITE_TYPES : Lists.newArrayList(WriteType.valueOf(mWriteType));
    List<OperationType> operations = mOperation == null ? Lists.newArrayList(OperationType.values()) : Lists.newArrayList(OperationType.valueOf(mOperation));
    for (ReadType readType : readTypes) {
        for (WriteType writeType : writeTypes) {
            for (OperationType opType : operations) {
                System.out.println(String.format("runTest --operation %s --readType %s --writeType %s", opType, readType, writeType));
                failed += runTest(opType, readType, writeType, fsContext);
            }
        }
    }
    if (failed > 0) {
        System.out.println("Number of failed tests: " + failed);
    }
    return failed;
}

Example 77 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FileInStreamRehydrationIntegrationTest method testRehydration.

@Test
public void testRehydration() throws Exception {
    FileSystem fs = FileSystem.Factory.create(ServerConfiguration.global());
    // Create a file with 10 blocks.
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    CreateFilePOptions op = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build();
    byte[] content = generateContent(BLOCK_BYTES * 10);
    try (FileOutStream os = fs.createFile(filePath, op)) {
        os.write(content);
    }
    // Validate file size and content.
    validateAndCloseFileStream(mFileSystem.openFile(filePath), content);
    // Grab the block master for emulating lost blocks.
    MasterProcess masterProcess = Whitebox.getInternalState(mLocalAlluxioClusterResource.get().getLocalAlluxioMaster(), "mMasterProcess");
    BlockMaster blockMaster = masterProcess.getMaster(BlockMaster.class);
    // Remove blocks externally from block-master.
    URIStatus status = fs.getStatus(filePath);
    blockMaster.removeBlocks(status.getBlockIds(), true);
    // Validate file size and content.
    validateAndCloseFileStream(mFileSystem.openFile(filePath), content);
}

Example 78 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FileOutStreamAsyncWriteIntegrationTest method testLostAsyncBlocks.

/**
 * Test eviction against a file that is slow to persist in Alluxio. The test cluster should be
 * configured with a high initial wait time, or -1
 *
 * This test performs the following actions:
 * - creates a file with ASYNC_THROUGH which fills the entire capacity of a worker
 * - Tries to create another 1-byte file on top and expects it to fail
 * - persists the file to the UFS
 * - Tries to create 1-byte file again and it should succeed this time
 */
private void testLostAsyncBlocks() throws Exception {
    long cap = FormatUtils.parseSpaceSize(TINY_WORKER_MEM);
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient();
    // Create a large-ish file in the UFS (relative to memory size)
    String p1 = "/test";
    FileSystemTestUtils.createByteFile(fs, p1, WritePType.ASYNC_THROUGH, (int) cap);
    URIStatus fstat = fs.listStatus(new AlluxioURI(p1)).get(0);
    int lostBlocks = fstat.getFileBlockInfos().stream().map(FileBlockInfo::getBlockInfo).filter(blk -> blk.getLocations().size() <= 0).mapToInt(blk -> 1).sum();
    assertEquals(cap, getClusterCapacity());
    assertEquals(cap, getUsedWorkerSpace());
    assertEquals(100, fstat.getInAlluxioPercentage());
    assertEquals(0, lostBlocks);
    // Try to create 1-byte file on top. Expect to fail.
    try {
        FileSystemTestUtils.createByteFile(fs, "/byte-file1", WritePType.MUST_CACHE, 1);
        assertTrue("Shouldn't reach here.", false);
    } catch (Exception e) {
    // expected.
    }
    FileSystemUtils.persistAndWait(fs, new AlluxioURI(p1), 0);
    fstat = fs.listStatus(new AlluxioURI(p1)).get(0);
    assertTrue(fstat.isPersisted());
    assertEquals(0, mLocalAlluxioClusterResource.get().getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class).getPinIdList().size());
    // Try to create 1-byte file on top. Expect to succeed.
    FileSystemTestUtils.createByteFile(fs, "/byte-file2", WritePType.MUST_CACHE, 1);
}

Example 79 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class S3RestServiceHandler method getBucket.

/**
 * @summary gets a bucket and lists all the objects in it
 * @param authorization header parameter authorization
 * @param bucket the bucket name
 * @param markerParam the optional marker param
 * @param prefixParam the optional prefix param
 * @param delimiterParam the optional delimiter param
 * @param encodingTypeParam optional encoding type param
 * @param maxKeysParam the optional max keys param
 * @param listTypeParam if listObjectV2 request
 * @param continuationTokenParam the optional continuationToken param for listObjectV2
 * @param startAfterParam  the optional startAfter param for listObjectV2
 * @return the response object
 */
@GET
@Path(BUCKET_PARAM)
public Response getBucket(@HeaderParam("Authorization") String authorization, @PathParam("bucket") final String bucket, @QueryParam("marker") final String markerParam, @QueryParam("prefix") final String prefixParam, @QueryParam("delimiter") final String delimiterParam, @QueryParam("encoding-type") final String encodingTypeParam, @QueryParam("max-keys") final int maxKeysParam, @QueryParam("list-type") final int listTypeParam, @QueryParam("continuation-token") final String continuationTokenParam, @QueryParam("start-after") final String startAfterParam) {
    return S3RestUtils.call(bucket, () -> {
        Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
        String marker = markerParam == null ? "" : markerParam;
        String prefix = prefixParam == null ? "" : prefixParam;
        String encodingType = encodingTypeParam == null ? "url" : encodingTypeParam;
        int maxKeys = maxKeysParam <= 0 ? ListBucketOptions.DEFAULT_MAX_KEYS : maxKeysParam;
        String continuationToken = continuationTokenParam == null ? "" : continuationTokenParam;
        String startAfter = startAfterParam == null ? "" : startAfterParam;
        ListBucketOptions listBucketOptions = ListBucketOptions.defaults().setMarker(marker).setPrefix(prefix).setMaxKeys(maxKeys).setDelimiter(delimiterParam).setEncodingType(encodingType).setListType(listTypeParam).setContinuationToken(continuationToken).setStartAfter(startAfter);
        String path = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        final FileSystem fs = getFileSystem(authorization);
        List<URIStatus> children;
        try {
            // only list the direct children if delimiter is not null
            if (delimiterParam != null) {
                path = parsePath(path, prefix, delimiterParam);
                children = fs.listStatus(new AlluxioURI(path));
            } else {
                ListStatusPOptions options = ListStatusPOptions.newBuilder().setRecursive(true).build();
                children = fs.listStatus(new AlluxioURI(path), options);
            }
        } catch (FileDoesNotExistException e) {
            // returned which does not match the S3 response behavior
            throw new S3Exception(e, bucket, S3ErrorCode.NO_SUCH_BUCKET);
        } catch (IOException | AlluxioException e) {
            throw new RuntimeException(e);
        }
        return new ListBucketResult(bucket, children, listBucketOptions);
    });
}

Example 80 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class S3RestServiceHandler method createBucket.

/**
 * @summary creates a bucket
 * @param authorization header parameter authorization
 * @param bucket the bucket name
 * @return the response object
 */
@PUT
@Path(BUCKET_PARAM)
public Response createBucket(@HeaderParam("Authorization") String authorization, @PathParam("bucket") final String bucket) {
    return S3RestUtils.call(bucket, () -> {
        Preconditions.checkNotNull(bucket, "required 'bucket' parameter is missing");
        String bucketPath = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
        final FileSystem fs = getFileSystem(authorization);
        // Create the bucket.
        CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setWriteType(S3RestUtils.getS3WriteType()).build();
        try {
            fs.createDirectory(new AlluxioURI(bucketPath), options);
        } catch (Exception e) {
            throw S3RestUtils.toBucketS3Exception(e, bucketPath);
        }
        return Response.Status.OK;
    });
}