Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 26 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FileOutStreamAsyncWriteIntegrationTest method asyncWriteNoEvictBeforeBlockCommit.

@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.USER_FILE_PERSISTENCE_INITIAL_WAIT_TIME, "-1", PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, "ASYNC_THROUGH", PropertyKey.Name.WORKER_RAMDISK_SIZE, TINY_WORKER_MEM, PropertyKey.Name.USER_BLOCK_SIZE_BYTES_DEFAULT, TINY_BLOCK_SIZE, PropertyKey.Name.USER_FILE_BUFFER_BYTES, TINY_BLOCK_SIZE, "alluxio.worker.tieredstore.level0.watermark.high.ratio", "0.5", "alluxio.worker.tieredstore.level0.watermark.low.ratio", "0.25" })
public void asyncWriteNoEvictBeforeBlockCommit() throws Exception {
    long writeSize = FormatUtils.parseSpaceSize(TINY_WORKER_MEM) - FormatUtils.parseSpaceSize(TINY_BLOCK_SIZE);
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient();
    AlluxioURI p1 = new AlluxioURI("/p1");
    FileOutStream fos = fs.createFile(p1, CreateFilePOptions.newBuilder().setWriteType(WritePType.ASYNC_THROUGH).setPersistenceWaitTime(-1).build());
    byte[] arr = new byte[(int) writeSize];
    Arrays.fill(arr, (byte) 0x7a);
    fos.write(arr);
    assertEquals(writeSize + FormatUtils.parseSpaceSize(TINY_BLOCK_SIZE), getClusterCapacity());
    // This will succeed.
    FileSystemTestUtils.createByteFile(fs, "/byte-file1", WritePType.MUST_CACHE, (int) FormatUtils.parseSpaceSize(TINY_BLOCK_SIZE));
    // This will not until the stream is closed and persisted.
    try {
        FileSystemTestUtils.createByteFile(fs, "/byte-file2", WritePType.MUST_CACHE, 2 * (int) FormatUtils.parseSpaceSize(TINY_BLOCK_SIZE));
        Assert.fail("Should have failed due to non-evictable block.");
    } catch (Exception e) {
    // expected.
    }
    fos.close();
    FileSystemUtils.persistAndWait(fs, p1, 0);
    // Now this should succeed.
    FileSystemTestUtils.createByteFile(fs, "/byte-file3", WritePType.MUST_CACHE, 2 * (int) FormatUtils.parseSpaceSize(TINY_BLOCK_SIZE));
}

Example 27 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ConcurrentFileSystemMasterUtils method unaryOperation.

/**
 * Helper for running concurrent operations. Enforces that the run time of this method is not
 * greater than the specified run time.
 *
 * @param fileSystem the filesystem to use
 * @param operation the operation to run concurrently
 * @param paths the paths to run the operation on
 * @param limitMs the maximum allowable run time, in ms
 * @return all exceptions encountered
 */
public static List<Throwable> unaryOperation(final FileSystem fileSystem, final UnaryOperation operation, final AlluxioURI[] paths, final long limitMs) throws Exception {
    final int numFiles = paths.length;
    final CyclicBarrier barrier = new CyclicBarrier(numFiles);
    List<Thread> threads = new ArrayList<>(numFiles);
    // If there are exceptions, we will store them here.
    final List<Throwable> errors = Collections.synchronizedList(new ArrayList<Throwable>());
    Thread.UncaughtExceptionHandler exceptionHandler = (th, ex) -> errors.add(ex);
    for (int i = 0; i < numFiles; i++) {
        final int iteration = i;
        Thread t = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    AuthenticatedClientUser.set(TEST_USER);
                    barrier.await();
                    switch(operation) {
                        case CREATE:
                            fileSystem.createFile(paths[iteration], sCreatePersistedFileOptions).close();
                            break;
                        case DELETE:
                            fileSystem.delete(paths[iteration]);
                            break;
                        case GET_FILE_INFO:
                            URIStatus status = fileSystem.getStatus(paths[iteration]);
                            if (!status.isFolder()) {
                                Assert.assertNotEquals(0, status.getBlockIds().size());
                            }
                            break;
                        case LIST_STATUS:
                            fileSystem.listStatus(paths[iteration]);
                            break;
                        default:
                            throw new IllegalArgumentException("'operation' is not a valid operation.");
                    }
                } catch (Exception e) {
                    Throwables.propagate(e);
                }
            }
        });
        t.setUncaughtExceptionHandler(exceptionHandler);
        threads.add(t);
    }
    Collections.shuffle(threads);
    long startMs = CommonUtils.getCurrentMs();
    for (Thread t : threads) {
        t.start();
    }
    for (Thread t : threads) {
        t.join();
    }
    long durationMs = CommonUtils.getCurrentMs() - startMs;
    Assert.assertTrue("Execution duration " + durationMs + " took longer than expected " + limitMs, durationMs < limitMs);
    return errors;
}

Example 28 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ConcurrentRecursiveCreateIntegrationTest method createDuringUfsRename.

@Test
public void createDuringUfsRename() throws Exception {
    FileSystem fs = mClusterResource.get().getClient();
    ExecutorService executor = Executors.newCachedThreadPool();
    UnderFileSystem ufs = Factory.createForRoot(ServerConfiguration.global());
    String ufsRoot = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    List<String> paths = new ArrayList<>();
    for (int i = 0; i < NUM_TOP_LEVEL_DIRS / 2; i++) {
        String alluxioPath = PathUtils.concatPath("/dir" + i, "a", "b", "c");
        ufs.mkdirs(PathUtils.concatPath(ufsRoot, alluxioPath));
        paths.add(alluxioPath);
    }
    executor.submit(new UfsRenamer(ufs, ufsRoot));
    for (int i = 0; i < 10; i++) {
        executor.submit(new AlluxioCreator(fs, paths));
    }
    CommonUtils.sleepMs(2 * Constants.SECOND_MS);
    executor.shutdownNow();
    assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
    mClusterResource.get().restartMasters();
    fs = mClusterResource.get().getClient();
    fs.listStatus(new AlluxioURI("/"));
}

Example 29 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class MultiWorkerIntegrationTest method readRecoverFromLostWorker.

@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.USER_SHORT_CIRCUIT_ENABLED, "false", PropertyKey.Name.USER_BLOCK_SIZE_BYTES_DEFAULT, "16MB", PropertyKey.Name.USER_STREAMING_READER_CHUNK_SIZE_BYTES, "64KB", PropertyKey.Name.USER_BLOCK_READ_RETRY_MAX_DURATION, "1s", PropertyKey.Name.WORKER_RAMDISK_SIZE, "1GB" })
public void readRecoverFromLostWorker() throws Exception {
    int offset = 17 * Constants.MB;
    int length = 33 * Constants.MB;
    int total = offset + length;
    // creates a test file on one worker
    AlluxioURI filePath = new AlluxioURI("/test");
    createFileOnWorker(total, filePath, mResource.get().getWorkerAddress());
    FileSystem fs = mResource.get().getClient();
    try (FileInStream in = fs.openFile(filePath, OpenFilePOptions.getDefaultInstance())) {
        byte[] buf = new byte[total];
        int size = in.read(buf, 0, offset);
        replicateFileBlocks(filePath);
        mResource.get().getWorkerProcess().stop();
        size += in.read(buf, offset, length);
        Assert.assertEquals(total, size);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(offset, size, buf));
    }
}

Example 30 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ClusterInitializationIntegrationTest method recoverClusterFail.

/**
 * When a user starts a cluster with journal logs, which are generated by previous running
 * cluster owned by a different user, it should fail and throw an exception.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.SECURITY_LOGIN_USERNAME, SUPER_USER })
public void recoverClusterFail() throws Exception {
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient();
    fs.createFile(new AlluxioURI("/testFile")).close();
    mLocalAlluxioClusterResource.get().stopFS();
    mThrown.expect(PermissionDeniedException.class);
    mThrown.expectMessage(ExceptionMessage.PERMISSION_DENIED.getMessage("Unauthorized user on root"));
    // user jack cannot recover master from journal, in which the root is owned by alluxio.
    MasterTestUtils.createLeaderFileSystemMasterFromJournal(new TestUserState(USER, ServerConfiguration.global())).close();
}