Examples with Future java.util.concurrent.Future used on opensource projects

Example 21 with Future

use of java.util.concurrent.Future in project hadoop by apache.

the class TestFSDownload method testDownloadBadPublic.

@Test(timeout = 10000)
public void testDownloadBadPublic() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    int size = 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
    Path path = new Path(basedir, "test-file");
    LocalResource rsrc = createFile(files, path, size, rand, vis);
    rsrcVis.put(rsrc, vis);
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    pending.put(rsrc, exec.submit(fsd));
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    Assert.assertTrue(pending.get(rsrc).isDone());
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            p.getValue().get();
            Assert.fail("We localized a file that is not public.");
        }
    } catch (ExecutionException e) {
        Assert.assertTrue(e.getCause() instanceof IOException);
    }
}

Example 22 with Future

use of java.util.concurrent.Future in project hadoop by apache.

the class TestFSDownload method testDirDownload.

@Test(timeout = 10000)
public void testDirDownload() throws IOException, InterruptedException {
    Configuration conf = new Configuration();
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    for (int i = 0; i < 5; ++i) {
        LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
        if (i % 2 == 1) {
            vis = LocalResourceVisibility.APPLICATION;
        }
        Path p = new Path(basedir, "dir" + i + ".jar");
        LocalResource rsrc = createJar(files, p, vis);
        rsrcVis.put(rsrc, vis);
        Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf);
        destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
        FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
        pending.put(rsrc, exec.submit(fsd));
    }
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    for (Future<Path> path : pending.values()) {
        Assert.assertTrue(path.isDone());
    }
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            Path localized = p.getValue().get();
            FileStatus status = files.getFileStatus(localized);
            System.out.println("Testing path " + localized);
            assert (status.isDirectory());
            assert (rsrcVis.containsKey(p.getKey()));
            verifyPermsRecursively(localized.getFileSystem(conf), files, localized, rsrcVis.get(p.getKey()));
        }
    } catch (ExecutionException e) {
        throw new IOException("Failed exec", e);
    }
}

Example 23 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class MultiThreadedClientExample method run.

@Override
public int run(String[] args) throws Exception {
    if (args.length < 1 || args.length > 2) {
        System.out.println("Usage: " + this.getClass().getName() + " tableName [num_operations]");
        return -1;
    }
    final TableName tableName = TableName.valueOf(args[0]);
    int numOperations = DEFAULT_NUM_OPERATIONS;
    // the second arg is the number of operations to send.
    if (args.length == 2) {
        numOperations = Integer.parseInt(args[1]);
    }
    // Threads for the client only.
    //
    // We don't want to mix hbase and business logic.
    //
    ExecutorService service = new ForkJoinPool(threads * 2);
    // Create two different connections showing how it's possible to
    // separate different types of requests onto different connections
    final Connection writeConnection = ConnectionFactory.createConnection(getConf(), service);
    final Connection readConnection = ConnectionFactory.createConnection(getConf(), service);
    // At this point the entire cache for the region locations is full.
    // Only do this if the number of regions in a table is easy to fit into memory.
    //
    // If you are interacting with more than 25k regions on a client then it's probably not good
    // to do this at all.
    warmUpConnectionCache(readConnection, tableName);
    warmUpConnectionCache(writeConnection, tableName);
    List<Future<Boolean>> futures = new ArrayList<>(numOperations);
    for (int i = 0; i < numOperations; i++) {
        double r = ThreadLocalRandom.current().nextDouble();
        Future<Boolean> f;
        // These callables are meant to represent real work done by your application.
        if (r < .30) {
            f = internalPool.submit(new WriteExampleCallable(writeConnection, tableName));
        } else if (r < .50) {
            f = internalPool.submit(new SingleWriteExampleCallable(writeConnection, tableName));
        } else {
            f = internalPool.submit(new ReadExampleCallable(writeConnection, tableName));
        }
        futures.add(f);
    }
    // Wait a long time for all the reads/writes to complete
    for (Future<Boolean> f : futures) {
        f.get(10, TimeUnit.MINUTES);
    }
    // Clean up after our selves for cleanliness
    internalPool.shutdownNow();
    service.shutdownNow();
    return 0;
}

Example 24 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class HFileReplicator method copyHFilesToStagingDir.

private Map<String, Path> copyHFilesToStagingDir() throws IOException {
    Map<String, Path> mapOfCopiedHFiles = new HashMap<>();
    Pair<byte[], List<String>> familyHFilePathsPair;
    List<String> hfilePaths;
    byte[] family;
    Path familyStagingDir;
    int familyHFilePathsPairsListSize;
    int totalNoOfHFiles;
    List<Pair<byte[], List<String>>> familyHFilePathsPairsList;
    FileSystem sourceFs = null;
    try {
        Path sourceClusterPath = new Path(sourceBaseNamespaceDirPath);
        /*
       * Path#getFileSystem will by default get the FS from cache. If both source and sink cluster
       * has same FS name service then it will return peer cluster FS. To avoid this we explicitly
       * disable the loading of FS from cache, so that a new FS is created with source cluster
       * configuration.
       */
        String sourceScheme = sourceClusterPath.toUri().getScheme();
        String disableCacheName = String.format("fs.%s.impl.disable.cache", new Object[] { sourceScheme });
        sourceClusterConf.setBoolean(disableCacheName, true);
        sourceFs = sourceClusterPath.getFileSystem(sourceClusterConf);
        User user = userProvider.getCurrent();
        // For each table name in the map
        for (Entry<String, List<Pair<byte[], List<String>>>> tableEntry : bulkLoadHFileMap.entrySet()) {
            String tableName = tableEntry.getKey();
            // Create staging directory for each table
            Path stagingDir = createStagingDir(hbaseStagingDir, user, TableName.valueOf(tableName));
            familyHFilePathsPairsList = tableEntry.getValue();
            familyHFilePathsPairsListSize = familyHFilePathsPairsList.size();
            // For each list of family hfile paths pair in the table
            for (int i = 0; i < familyHFilePathsPairsListSize; i++) {
                familyHFilePathsPair = familyHFilePathsPairsList.get(i);
                family = familyHFilePathsPair.getFirst();
                hfilePaths = familyHFilePathsPair.getSecond();
                familyStagingDir = new Path(stagingDir, Bytes.toString(family));
                totalNoOfHFiles = hfilePaths.size();
                // For each list of hfile paths for the family
                List<Future<Void>> futures = new ArrayList<>();
                Callable<Void> c;
                Future<Void> future;
                int currentCopied = 0;
                // Copy the hfiles parallely
                while (totalNoOfHFiles > currentCopied + this.copiesPerThread) {
                    c = new Copier(sourceFs, familyStagingDir, hfilePaths.subList(currentCopied, currentCopied + this.copiesPerThread));
                    future = exec.submit(c);
                    futures.add(future);
                    currentCopied += this.copiesPerThread;
                }
                int remaining = totalNoOfHFiles - currentCopied;
                if (remaining > 0) {
                    c = new Copier(sourceFs, familyStagingDir, hfilePaths.subList(currentCopied, currentCopied + remaining));
                    future = exec.submit(c);
                    futures.add(future);
                }
                for (Future<Void> f : futures) {
                    try {
                        f.get();
                    } catch (InterruptedException e) {
                        InterruptedIOException iioe = new InterruptedIOException("Failed to copy HFiles to local file system. This will be retried again " + "by the source cluster.");
                        iioe.initCause(e);
                        throw iioe;
                    } catch (ExecutionException e) {
                        throw new IOException("Failed to copy HFiles to local file system. This will " + "be retried again by the source cluster.", e);
                    }
                }
            }
            // Add the staging directory to this table. Staging directory contains all the hfiles
            // belonging to this table
            mapOfCopiedHFiles.put(tableName, stagingDir);
        }
        return mapOfCopiedHFiles;
    } finally {
        if (sourceFs != null) {
            sourceFs.close();
        }
        if (exec != null) {
            exec.shutdown();
        }
    }
}

Example 25 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class HFileCorruptionChecker method checkTableDir.

/**
   * Check all the regiondirs in the specified tableDir
   *
   * @param tableDir
   *          path to a table
   * @throws IOException
   */
void checkTableDir(Path tableDir) throws IOException {
    List<FileStatus> rds = FSUtils.listStatusWithStatusFilter(fs, tableDir, new RegionDirFilter(fs));
    if (rds == null) {
        if (!fs.exists(tableDir)) {
            LOG.warn("Table Directory " + tableDir + " does not exist.  Likely due to concurrent delete. Skipping.");
            missing.add(tableDir);
        }
        return;
    }
    // Parallelize check at the region dir level
    List<RegionDirChecker> rdcs = new ArrayList<>(rds.size() + 1);
    List<Future<Void>> rdFutures;
    for (FileStatus rdFs : rds) {
        Path rdDir = rdFs.getPath();
        RegionDirChecker work = new RegionDirChecker(rdDir);
        rdcs.add(work);
    }
    // add mob region
    rdcs.add(createMobRegionDirChecker(tableDir));
    // Submit and wait for completion
    try {
        rdFutures = executor.invokeAll(rdcs);
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        LOG.warn("Region dirs checking interrupted!", ie);
        return;
    }
    for (int i = 0; i < rdFutures.size(); i++) {
        Future<Void> f = rdFutures.get(i);
        try {
            f.get();
        } catch (ExecutionException e) {
            LOG.warn("Failed to quarantine an HFile in regiondir " + rdcs.get(i).regionDir, e.getCause());
            // rethrow IOExceptions
            if (e.getCause() instanceof IOException) {
                throw (IOException) e.getCause();
            }
            // rethrow RuntimeExceptions
            if (e.getCause() instanceof RuntimeException) {
                throw (RuntimeException) e.getCause();
            }
            // this should never happen
            LOG.error("Unexpected exception encountered", e);
            // bailing out.
            return;
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            LOG.warn("Region dirs check interrupted!", ie);
            // bailing out
            return;
        }
    }
}