Examples with ConcurrentHashSet alluxio.collections.ConcurrentHashSet used on opensource projects

Example 6 with ConcurrentHashSet

use of alluxio.collections.ConcurrentHashSet in project alluxio by Alluxio.

the class SupportedHdfsActiveSyncProvider method recordFileChanged.

private void recordFileChanged(String syncPoint, String filePath, long txId) {
    AlluxioURI syncPointUri = new AlluxioURI(syncPoint);
    mChangedFiles.computeIfAbsent(syncPoint, (key) -> {
        mActivity.put(syncPoint, 0);
        mAge.put(syncPoint, 0);
        mTxIdMap.put(syncPoint, txId);
        return new ConcurrentHashSet<>();
    });
    try (LockResource r = new LockResource(mWriteLock)) {
        mChangedFiles.get(syncPoint).add(new AlluxioURI(syncPointUri.getScheme(), syncPointUri.getAuthority(), filePath));
        mActivity.put(syncPoint, mActivity.get(syncPoint) + 1);
    }
}

Example 7 with ConcurrentHashSet

use of alluxio.collections.ConcurrentHashSet in project alluxio by Alluxio.

the class BlockLockManagerTest method stress.

/**
 * Tests that taking and releasing many block locks concurrently won't cause a failure.
 *
 * This is done by creating 200 threads, 100 for each of 2 different block ids. Each thread locks
 * and then unlocks its block 50 times. After this, it takes a final lock on its block before
 * returning. At the end of the test, the internal state of the lock manager is validated.
 */
@Test(timeout = 10000)
public void stress() throws Throwable {
    final int numBlocks = 2;
    final int threadsPerBlock = 100;
    final int lockUnlocksPerThread = 50;
    setMaxLocks(numBlocks);
    final BlockLockManager manager = new BlockLockManager();
    final List<Thread> threads = new ArrayList<>();
    final CyclicBarrier barrier = new CyclicBarrier(numBlocks * threadsPerBlock);
    // If there are exceptions, we will store them here.
    final ConcurrentHashSet<Throwable> failedThreadThrowables = new ConcurrentHashSet<>();
    Thread.UncaughtExceptionHandler exceptionHandler = new Thread.UncaughtExceptionHandler() {

        public void uncaughtException(Thread th, Throwable ex) {
            failedThreadThrowables.add(ex);
        }
    };
    for (int blockId = 0; blockId < numBlocks; blockId++) {
        final int finalBlockId = blockId;
        for (int i = 0; i < threadsPerBlock; i++) {
            Thread t = new Thread(new Runnable() {

                @Override
                public void run() {
                    try {
                        barrier.await();
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    // Lock and unlock the block lockUnlocksPerThread times.
                    for (int j = 0; j < lockUnlocksPerThread; j++) {
                        long lockId = manager.lockBlock(TEST_SESSION_ID, finalBlockId, BlockLockType.READ);
                        assertTrue(manager.unlockBlockNoException(lockId));
                    }
                    // Lock the block one last time.
                    manager.lockBlock(TEST_SESSION_ID, finalBlockId, BlockLockType.READ);
                }
            });
            t.setUncaughtExceptionHandler(exceptionHandler);
            threads.add(t);
        }
    }
    Collections.shuffle(threads);
    for (Thread t : threads) {
        t.start();
    }
    for (Thread t : threads) {
        t.join();
    }
    if (!failedThreadThrowables.isEmpty()) {
        StringBuilder sb = new StringBuilder("Failed with the following errors:\n");
        for (Throwable failedThreadThrowable : failedThreadThrowables) {
            sb.append(Throwables.getStackTraceAsString(failedThreadThrowable));
        }
        Assert.fail(sb.toString());
    }
    manager.validate();
}

Example 8 with ConcurrentHashSet

use of alluxio.collections.ConcurrentHashSet in project alluxio by Alluxio.

the class CheckConsistencyCommand method runConsistencyCheck.

/**
 * Checks the inconsistent files and directories which exist in Alluxio but don't exist in the
 * under storage, repairs the inconsistent paths by deleting them if repairConsistency is true.
 *
 * @param path the specified path to be checked
 * @param repairConsistency whether to repair the consistency or not
 * @throws AlluxioException
 * @throws IOException
 */
private void runConsistencyCheck(AlluxioURI path, boolean repairConsistency, int repairThreads) throws AlluxioException, IOException {
    List<AlluxioURI> inconsistentUris = checkConsistency(path, FileSystemOptions.checkConsistencyDefaults(mFsContext.getPathConf(path)));
    if (inconsistentUris.isEmpty()) {
        System.out.println(path + " is consistent with the under storage system.");
        return;
    }
    if (!repairConsistency) {
        Collections.sort(inconsistentUris);
        System.out.println("The following files are inconsistent:");
        for (AlluxioURI uri : inconsistentUris) {
            System.out.println(uri);
        }
    } else {
        Collections.sort(inconsistentUris);
        System.out.println(String.format("%s has: %d inconsistent files. Repairing with %d threads.", path, inconsistentUris.size(), repairThreads));
        ConcurrentHashSet<AlluxioURI> inconsistentDirs = new ConcurrentHashSet<>();
        ExecutorService svc = Executors.newFixedThreadPool(repairThreads);
        CompletionService<Boolean> completionService = new ExecutorCompletionService<>(svc);
        ConcurrentHashSet<Exception> exceptions = new ConcurrentHashSet<>();
        int totalUris = inconsistentUris.size();
        for (AlluxioURI inconsistentUri : inconsistentUris) {
            completionService.submit(() -> {
                try {
                    URIStatus status = mFileSystem.getStatus(inconsistentUri);
                    if (status.isFolder()) {
                        inconsistentDirs.add(inconsistentUri);
                        return;
                    }
                    System.out.println("repairing path: " + inconsistentUri);
                    DeletePOptions deleteOptions = DeletePOptions.newBuilder().setAlluxioOnly(true).build();
                    mFileSystem.delete(inconsistentUri, deleteOptions);
                    mFileSystem.exists(inconsistentUri);
                    System.out.println(inconsistentUri + " repaired");
                    System.out.println();
                } catch (AlluxioException | IOException e) {
                    exceptions.add(e);
                }
            }, true);
        }
        waitForTasks(completionService, totalUris, exceptions);
        int totalDirs = inconsistentDirs.size();
        for (AlluxioURI uri : inconsistentDirs) {
            completionService.submit(() -> {
                try {
                    DeletePOptions deleteOptions = DeletePOptions.newBuilder().setAlluxioOnly(true).setRecursive(true).build();
                    System.out.println("repairing path: " + uri);
                    mFileSystem.delete(uri, deleteOptions);
                    mFileSystem.exists(uri);
                    System.out.println(uri + "repaired");
                    System.out.println();
                } catch (AlluxioException | IOException e) {
                    exceptions.add(e);
                }
            }, true);
        }
        waitForTasks(completionService, totalDirs, exceptions);
        svc.shutdown();
    }
}

Example 9 with ConcurrentHashSet

use of alluxio.collections.ConcurrentHashSet in project alluxio by Alluxio.

the class ConcurrentFileSystemMasterTest method consistentGetFileInfo.

/**
   * Tests that getFileInfo (read operation) either returns the correct file info or fails if it
   * has been renamed while the operation was waiting for the file lock.
   */
@Test
public void consistentGetFileInfo() throws Exception {
    final int iterations = CONCURRENCY_FACTOR;
    final AlluxioURI file = new AlluxioURI("/file");
    final AlluxioURI dst = new AlluxioURI("/dst");
    final CyclicBarrier barrier = new CyclicBarrier(2);
    // If there are exceptions, we will store them here.
    final ConcurrentHashSet<Throwable> errors = new ConcurrentHashSet<>();
    Thread.UncaughtExceptionHandler exceptionHandler = new Thread.UncaughtExceptionHandler() {

        public void uncaughtException(Thread th, Throwable ex) {
            errors.add(ex);
        }
    };
    for (int i = 0; i < iterations; i++) {
        // Don't want sleeping ufs behavior, so do not write to ufs
        mFileSystem.createFile(file, CreateFileOptions.defaults().setWriteType(WriteType.MUST_CACHE)).close();
        Thread renamer = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    AuthenticatedClientUser.set(TEST_USER);
                    barrier.await();
                    mFileSystem.rename(file, dst);
                    mFileSystem.delete(dst);
                } catch (Exception e) {
                    Assert.fail(e.getMessage());
                }
            }
        });
        renamer.setUncaughtExceptionHandler(exceptionHandler);
        Thread reader = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    AuthenticatedClientUser.set(TEST_USER);
                    barrier.await();
                    URIStatus status = mFileSystem.getStatus(file);
                    // If the uri status is successfully obtained, then the path should match
                    Assert.assertEquals(file.getName(), status.getName());
                } catch (InvalidPathException | FileDoesNotExistException e) {
                // InvalidPathException - if the file is renamed while the thread waits for the lock.
                // FileDoesNotExistException - if the file is fully renamed before the getFileInfo call.
                } catch (Exception e) {
                    Assert.fail(e.getMessage());
                }
            }
        });
        reader.setUncaughtExceptionHandler(exceptionHandler);
        renamer.start();
        reader.start();
        renamer.join();
        reader.join();
        Assert.assertTrue("Errors detected: " + errors, errors.isEmpty());
    }
}

Example 10 with ConcurrentHashSet

use of alluxio.collections.ConcurrentHashSet in project alluxio by Alluxio.

the class ConcurrentRenameIntegrationTest method consistentGetFileInfo.

/**
 * Tests that getFileInfo (read operation) either returns the correct file info or fails if it
 * has been renamed while the operation was waiting for the file lock.
 */
@Test
public void consistentGetFileInfo() throws Exception {
    final int iterations = CONCURRENCY_FACTOR;
    final AlluxioURI file = new AlluxioURI("/file");
    final AlluxioURI dst = new AlluxioURI("/dst");
    final CyclicBarrier barrier = new CyclicBarrier(2);
    // If there are exceptions, we will store them here.
    final ConcurrentHashSet<Throwable> errors = new ConcurrentHashSet<>();
    Thread.UncaughtExceptionHandler exceptionHandler = new Thread.UncaughtExceptionHandler() {

        public void uncaughtException(Thread th, Throwable ex) {
            errors.add(ex);
        }
    };
    for (int i = 0; i < iterations; i++) {
        // Don't want sleeping ufs behavior, so do not write to ufs
        mFileSystem.createFile(file, CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build()).close();
        Thread renamer = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    AuthenticatedClientUser.set(TEST_USER);
                    barrier.await();
                    mFileSystem.rename(file, dst);
                    mFileSystem.delete(dst);
                } catch (Exception e) {
                    Assert.fail(e.getMessage());
                }
            }
        });
        renamer.setUncaughtExceptionHandler(exceptionHandler);
        Thread reader = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    AuthenticatedClientUser.set(TEST_USER);
                    barrier.await();
                    URIStatus status = mFileSystem.getStatus(file);
                    // If the uri status is successfully obtained, then the path should match
                    Assert.assertEquals(file.getName(), status.getName());
                } catch (AlluxioException e) {
                // AlluxioException - if the file is renamed while the thread waits for the lock.
                // FileDoesNotExistException - if the file is fully renamed before the getFileInfo call.
                } catch (Exception e) {
                    Assert.fail(e.getMessage());
                }
            }
        });
        reader.setUncaughtExceptionHandler(exceptionHandler);
        renamer.start();
        reader.start();
        renamer.join();
        reader.join();
        Assert.assertTrue("Errors detected: " + errors, errors.isEmpty());
    }
}