Examples with Callable java.util.concurrent.Callable used on opensource projects

Example 26 with Callable

use of java.util.concurrent.Callable in project hbase by apache.

the class PartitionedMobCompactor method compactMobFiles.

/**
   * Compacts the selected small mob files and all the del files.
   * @param request The compaction request.
   * @return The paths of new mob files after compactions.
   * @throws IOException if IO failure is encountered
   */
protected List<Path> compactMobFiles(final PartitionedMobCompactionRequest request) throws IOException {
    Collection<CompactionPartition> partitions = request.compactionPartitions;
    if (partitions == null || partitions.isEmpty()) {
        LOG.info("No partitions of mob files");
        return Collections.emptyList();
    }
    List<Path> paths = new ArrayList<>();
    final Connection c = ConnectionFactory.createConnection(conf);
    final Table table = c.getTable(tableName);
    try {
        Map<CompactionPartitionId, Future<List<Path>>> results = new HashMap<>();
        // compact the mob files by partitions in parallel.
        for (final CompactionPartition partition : partitions) {
            // How to efficiently come up a list of delFiles for one partition?
            // Search the delPartitions and collect all the delFiles for the partition
            // One optimization can do is that if there is no del file, we do not need to
            // come up with startKey/endKey.
            List<StoreFile> delFiles = getListOfDelFilesForPartition(partition, request.getDelPartitions());
            results.put(partition.getPartitionId(), pool.submit(new Callable<List<Path>>() {

                @Override
                public List<Path> call() throws Exception {
                    LOG.info("Compacting mob files for partition " + partition.getPartitionId());
                    return compactMobFilePartition(request, partition, delFiles, c, table);
                }
            }));
        }
        // compact the partitions in parallel.
        List<CompactionPartitionId> failedPartitions = new ArrayList<>();
        for (Entry<CompactionPartitionId, Future<List<Path>>> result : results.entrySet()) {
            try {
                paths.addAll(result.getValue().get());
            } catch (Exception e) {
                // just log the error
                LOG.error("Failed to compact the partition " + result.getKey(), e);
                failedPartitions.add(result.getKey());
            }
        }
        if (!failedPartitions.isEmpty()) {
            // if any partition fails in the compaction, directly throw an exception.
            throw new IOException("Failed to compact the partitions " + failedPartitions);
        }
    } finally {
        try {
            table.close();
        } catch (IOException e) {
            LOG.error("Failed to close the Table", e);
        }
    }
    return paths;
}

Example 27 with Callable

use of java.util.concurrent.Callable in project hadoop by apache.

the class TestNMTimelineCollectorManager method testMultithreadedAddAndRemove.

@Test
public void testMultithreadedAddAndRemove() throws Exception {
    final int numApps = 5;
    List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
    for (int i = 0; i < numApps; i++) {
        final ApplicationId appId = ApplicationId.newInstance(0L, i);
        Callable<Boolean> task = new Callable<Boolean>() {

            public Boolean call() {
                AppLevelTimelineCollector collector = new AppLevelTimelineCollector(appId);
                boolean successPut = (collectorManager.putIfAbsent(appId, collector) == collector);
                return successPut && collectorManager.remove(appId);
            }
        };
        tasks.add(task);
    }
    ExecutorService executor = Executors.newFixedThreadPool(numApps);
    try {
        List<Future<Boolean>> futures = executor.invokeAll(tasks);
        for (Future<Boolean> future : futures) {
            assertTrue(future.get());
        }
    } finally {
        executor.shutdownNow();
    }
    // check the keys
    for (int i = 0; i < numApps; i++) {
        final ApplicationId appId = ApplicationId.newInstance(0L, i);
        assertFalse(collectorManager.containsTimelineCollector(appId));
    }
}

Example 28 with Callable

use of java.util.concurrent.Callable in project hbase by apache.

the class HBaseAdmin method compact.

private void compact(final ServerName sn, final HRegionInfo hri, final boolean major, final byte[] family) throws IOException {
    final AdminService.BlockingInterface admin = this.connection.getAdmin(sn);
    Callable<Void> callable = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            // TODO: There is no timeout on this controller. Set one!
            HBaseRpcController controller = rpcControllerFactory.newController();
            CompactRegionRequest request = RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major, family);
            admin.compactRegion(controller, request);
            return null;
        }
    };
    ProtobufUtil.call(callable);
}

Example 29 with Callable

use of java.util.concurrent.Callable in project hbase by apache.

the class HBaseAdmin method flushRegion.

@Override
public void flushRegion(final byte[] regionName) throws IOException {
    Pair<HRegionInfo, ServerName> regionServerPair = getRegion(regionName);
    if (regionServerPair == null) {
        throw new IllegalArgumentException("Unknown regionname: " + Bytes.toStringBinary(regionName));
    }
    if (regionServerPair.getSecond() == null) {
        throw new NoServerForRegionException(Bytes.toStringBinary(regionName));
    }
    final HRegionInfo hRegionInfo = regionServerPair.getFirst();
    ServerName serverName = regionServerPair.getSecond();
    final AdminService.BlockingInterface admin = this.connection.getAdmin(serverName);
    Callable<Void> callable = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            // TODO: There is no timeout on this controller. Set one!
            HBaseRpcController controller = rpcControllerFactory.newController();
            FlushRegionRequest request = RequestConverter.buildFlushRegionRequest(hRegionInfo.getRegionName());
            admin.flushRegion(controller, request);
            return null;
        }
    };
    ProtobufUtil.call(callable);
}

Example 30 with Callable

use of java.util.concurrent.Callable in project hbase by apache.

the class LoadIncrementalHFiles method bulkLoadPhase.

/**
   * This takes the LQI's grouped by likely regions and attempts to bulk load
   * them.  Any failures are re-queued for another pass with the
   * groupOrSplitPhase.
   */
protected void bulkLoadPhase(final Table table, final Connection conn, ExecutorService pool, Deque<LoadQueueItem> queue, final Multimap<ByteBuffer, LoadQueueItem> regionGroups, boolean copyFile, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
    // atomically bulk load the groups.
    Set<Future<List<LoadQueueItem>>> loadingFutures = new HashSet<>();
    for (Entry<ByteBuffer, ? extends Collection<LoadQueueItem>> e : regionGroups.asMap().entrySet()) {
        final byte[] first = e.getKey().array();
        final Collection<LoadQueueItem> lqis = e.getValue();
        final ClientServiceCallable<byte[]> serviceCallable = buildClientServiceCallable(conn, table.getName(), first, lqis, copyFile);
        final Callable<List<LoadQueueItem>> call = new Callable<List<LoadQueueItem>>() {

            @Override
            public List<LoadQueueItem> call() throws Exception {
                List<LoadQueueItem> toRetry = tryAtomicRegionLoad(serviceCallable, table.getName(), first, lqis);
                return toRetry;
            }
        };
        if (item2RegionMap != null) {
            for (LoadQueueItem lqi : lqis) {
                item2RegionMap.put(lqi, e.getKey());
            }
        }
        loadingFutures.add(pool.submit(call));
    }
    // get all the results.
    for (Future<List<LoadQueueItem>> future : loadingFutures) {
        try {
            List<LoadQueueItem> toRetry = future.get();
            if (item2RegionMap != null) {
                for (LoadQueueItem lqi : toRetry) {
                    item2RegionMap.remove(lqi);
                }
            }
            // LQIs that are requeued to be regrouped.
            queue.addAll(toRetry);
        } catch (ExecutionException e1) {
            Throwable t = e1.getCause();
            if (t instanceof IOException) {
                // TODO Implement bulk load recovery
                throw new IOException("BulkLoad encountered an unrecoverable problem", t);
            }
            LOG.error("Unexpected execution exception during bulk load", e1);
            throw new IllegalStateException(t);
        } catch (InterruptedException e1) {
            LOG.error("Unexpected interrupted exception during bulk load", e1);
            throw (InterruptedIOException) new InterruptedIOException().initCause(e1);
        }
    }
}