Examples with Future java.util.concurrent.Future used on opensource projects

Example 31 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class LoadIncrementalHFiles method bulkLoadPhase.

/**
   * This takes the LQI's grouped by likely regions and attempts to bulk load
   * them.  Any failures are re-queued for another pass with the
   * groupOrSplitPhase.
   */
protected void bulkLoadPhase(final Table table, final Connection conn, ExecutorService pool, Deque<LoadQueueItem> queue, final Multimap<ByteBuffer, LoadQueueItem> regionGroups, boolean copyFile, Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
    // atomically bulk load the groups.
    Set<Future<List<LoadQueueItem>>> loadingFutures = new HashSet<>();
    for (Entry<ByteBuffer, ? extends Collection<LoadQueueItem>> e : regionGroups.asMap().entrySet()) {
        final byte[] first = e.getKey().array();
        final Collection<LoadQueueItem> lqis = e.getValue();
        final ClientServiceCallable<byte[]> serviceCallable = buildClientServiceCallable(conn, table.getName(), first, lqis, copyFile);
        final Callable<List<LoadQueueItem>> call = new Callable<List<LoadQueueItem>>() {

            @Override
            public List<LoadQueueItem> call() throws Exception {
                List<LoadQueueItem> toRetry = tryAtomicRegionLoad(serviceCallable, table.getName(), first, lqis);
                return toRetry;
            }
        };
        if (item2RegionMap != null) {
            for (LoadQueueItem lqi : lqis) {
                item2RegionMap.put(lqi, e.getKey());
            }
        }
        loadingFutures.add(pool.submit(call));
    }
    // get all the results.
    for (Future<List<LoadQueueItem>> future : loadingFutures) {
        try {
            List<LoadQueueItem> toRetry = future.get();
            if (item2RegionMap != null) {
                for (LoadQueueItem lqi : toRetry) {
                    item2RegionMap.remove(lqi);
                }
            }
            // LQIs that are requeued to be regrouped.
            queue.addAll(toRetry);
        } catch (ExecutionException e1) {
            Throwable t = e1.getCause();
            if (t instanceof IOException) {
                // TODO Implement bulk load recovery
                throw new IOException("BulkLoad encountered an unrecoverable problem", t);
            }
            LOG.error("Unexpected execution exception during bulk load", e1);
            throw new IllegalStateException(t);
        } catch (InterruptedException e1) {
            LOG.error("Unexpected interrupted exception during bulk load", e1);
            throw (InterruptedIOException) new InterruptedIOException().initCause(e1);
        }
    }
}

Example 32 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class RegionMover method unloadRegions.

@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DLS_DEAD_LOCAL_STORE", justification = "FB is wrong; its size is read")
private void unloadRegions(Admin admin, String server, ArrayList<String> regionServers, boolean ack, List<HRegionInfo> movedRegions) throws Exception {
    // FindBugs: DLS_DEAD_LOCAL_STORE
    List<HRegionInfo> regionsToMove = new ArrayList<>();
    regionsToMove = getRegions(this.conf, server);
    if (regionsToMove.isEmpty()) {
        LOG.info("No Regions to move....Quitting now");
        return;
    } else if (regionServers.isEmpty()) {
        LOG.warn("No Regions were moved - no servers available");
        throw new Exception("No online region servers");
    }
    while (true) {
        regionsToMove = getRegions(this.conf, server);
        regionsToMove.removeAll(movedRegions);
        if (regionsToMove.isEmpty()) {
            break;
        }
        int counter = 0;
        LOG.info("Moving " + regionsToMove.size() + " regions from " + this.hostname + " to " + regionServers.size() + " servers using " + this.maxthreads + " threads .Ack Mode:" + ack);
        ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads);
        List<Future<Boolean>> taskList = new ArrayList<>();
        int serverIndex = 0;
        while (counter < regionsToMove.size()) {
            if (ack) {
                Future<Boolean> task = moveRegionsPool.submit(new MoveWithAck(admin, regionsToMove.get(counter), server, regionServers.get(serverIndex), movedRegions));
                taskList.add(task);
            } else {
                Future<Boolean> task = moveRegionsPool.submit(new MoveWithoutAck(admin, regionsToMove.get(counter), server, regionServers.get(serverIndex), movedRegions));
                taskList.add(task);
            }
            counter++;
            serverIndex = (serverIndex + 1) % regionServers.size();
        }
        moveRegionsPool.shutdown();
        long timeoutInSeconds = regionsToMove.size() * admin.getConfiguration().getInt(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX);
        try {
            if (!moveRegionsPool.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS)) {
                moveRegionsPool.shutdownNow();
            }
        } catch (InterruptedException e) {
            moveRegionsPool.shutdownNow();
            Thread.currentThread().interrupt();
        }
        for (Future<Boolean> future : taskList) {
            try {
                // if even after shutdownNow threads are stuck we wait for 5 secs max
                if (!future.get(5, TimeUnit.SECONDS)) {
                    LOG.error("Was Not able to move region....Exiting Now");
                    throw new Exception("Could not move region Exception");
                }
            } catch (InterruptedException e) {
                LOG.error("Interrupted while waiting for Thread to Complete " + e.getMessage(), e);
                throw e;
            } catch (ExecutionException e) {
                LOG.error("Got Exception From Thread While moving region " + e.getMessage(), e);
                throw e;
            } catch (CancellationException e) {
                LOG.error("Thread for moving region cancelled. Timeout for cancellation:" + timeoutInSeconds + "secs", e);
                throw e;
            }
        }
    }
}

Example 33 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class RegionMover method loadRegions.

private void loadRegions(Admin admin, String hostname, int port, List<HRegionInfo> regionsToMove, boolean ack) throws Exception {
    String server = null;
    List<HRegionInfo> movedRegions = Collections.synchronizedList(new ArrayList<HRegionInfo>());
    int maxWaitInSeconds = admin.getConfiguration().getInt(SERVERSTART_WAIT_MAX_KEY, DEFAULT_SERVERSTART_WAIT_MAX);
    long maxWait = EnvironmentEdgeManager.currentTime() + maxWaitInSeconds * 1000;
    while ((EnvironmentEdgeManager.currentTime() < maxWait) && (server == null)) {
        try {
            ArrayList<String> regionServers = getServers(admin);
            // Remove the host Region server from target Region Servers list
            server = stripServer(regionServers, hostname, port);
            if (server != null) {
                break;
            }
        } catch (IOException e) {
            LOG.warn("Could not get list of region servers", e);
        } catch (Exception e) {
            LOG.info("hostname=" + hostname + " is not up yet, waiting");
        }
        try {
            Thread.sleep(500);
        } catch (InterruptedException e) {
            LOG.error("Interrupted while waiting for " + hostname + " to be up.Quitting now", e);
            throw e;
        }
    }
    if (server == null) {
        LOG.error("Host:" + hostname + " is not up.Giving up.");
        throw new Exception("Host to load regions not online");
    }
    LOG.info("Moving " + regionsToMove.size() + " regions to " + server + " using " + this.maxthreads + " threads.Ack mode:" + this.ack);
    ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads);
    List<Future<Boolean>> taskList = new ArrayList<>();
    int counter = 0;
    while (counter < regionsToMove.size()) {
        HRegionInfo region = regionsToMove.get(counter);
        String currentServer = getServerNameForRegion(admin, region);
        if (currentServer == null) {
            LOG.warn("Could not get server for Region:" + region.getEncodedName() + " moving on");
            counter++;
            continue;
        } else if (server.equals(currentServer)) {
            LOG.info("Region " + region.getRegionNameAsString() + "already on target server=" + server);
            counter++;
            continue;
        }
        if (ack) {
            Future<Boolean> task = moveRegionsPool.submit(new MoveWithAck(admin, region, currentServer, server, movedRegions));
            taskList.add(task);
        } else {
            Future<Boolean> task = moveRegionsPool.submit(new MoveWithoutAck(admin, region, currentServer, server, movedRegions));
            taskList.add(task);
        }
        counter++;
    }
    moveRegionsPool.shutdown();
    long timeoutInSeconds = regionsToMove.size() * admin.getConfiguration().getInt(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX);
    try {
        if (!moveRegionsPool.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS)) {
            moveRegionsPool.shutdownNow();
        }
    } catch (InterruptedException e) {
        moveRegionsPool.shutdownNow();
        Thread.currentThread().interrupt();
    }
    for (Future<Boolean> future : taskList) {
        try {
            // if even after shutdownNow threads are stuck we wait for 5 secs max
            if (!future.get(5, TimeUnit.SECONDS)) {
                LOG.error("Was Not able to move region....Exiting Now");
                throw new Exception("Could not move region Exception");
            }
        } catch (InterruptedException e) {
            LOG.error("Interrupted while waiting for Thread to Complete " + e.getMessage(), e);
            throw e;
        } catch (ExecutionException e) {
            LOG.error("Got Exception From Thread While moving region " + e.getMessage(), e);
            throw e;
        } catch (CancellationException e) {
            LOG.error("Thread for moving region cancelled. Timeout for cancellation:" + timeoutInSeconds + "secs", e);
            throw e;
        }
    }
}

Example 34 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class HBaseFsck method processRegionServers.

/**
   * Contacts each regionserver and fetches metadata about regions.
   * @param regionServerList - the list of region servers to connect to
   * @throws IOException if a remote or network exception occurs
   */
void processRegionServers(Collection<ServerName> regionServerList) throws IOException, InterruptedException {
    List<WorkItemRegion> workItems = new ArrayList<>(regionServerList.size());
    List<Future<Void>> workFutures;
    // loop to contact each region server in parallel
    for (ServerName rsinfo : regionServerList) {
        workItems.add(new WorkItemRegion(this, rsinfo, errors, connection));
    }
    workFutures = executor.invokeAll(workItems);
    for (int i = 0; i < workFutures.size(); i++) {
        WorkItemRegion item = workItems.get(i);
        Future<Void> f = workFutures.get(i);
        try {
            f.get();
        } catch (ExecutionException e) {
            LOG.warn("Could not process regionserver " + item.rsinfo.getHostAndPort(), e.getCause());
        }
    }
}

Example 35 with Future

use of java.util.concurrent.Future in project hbase by apache.

the class TestAsyncTableGetMultiThreaded method test.

@Test
public void test() throws IOException, InterruptedException, ExecutionException {
    int numThreads = 20;
    AtomicBoolean stop = new AtomicBoolean(false);
    ExecutorService executor = Executors.newFixedThreadPool(numThreads, Threads.newDaemonThreadFactory("TestAsyncGet-"));
    List<Future<?>> futures = new ArrayList<>();
    IntStream.range(0, numThreads).forEach(i -> futures.add(executor.submit(() -> {
        run(stop);
        return null;
    })));
    Collections.shuffle(Arrays.asList(SPLIT_KEYS), new Random(123));
    Admin admin = TEST_UTIL.getAdmin();
    for (byte[] splitPoint : SPLIT_KEYS) {
        admin.split(TABLE_NAME, splitPoint);
        for (HRegion region : TEST_UTIL.getHBaseCluster().getRegions(TABLE_NAME)) {
            region.compact(true);
        }
        Thread.sleep(5000);
        admin.balancer(true);
        Thread.sleep(5000);
        ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
        ServerName newMetaServer = TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer().getServerName()).filter(s -> !s.equals(metaServer)).findAny().get();
        admin.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), Bytes.toBytes(newMetaServer.getServerName()));
        Thread.sleep(5000);
    }
    stop.set(true);
    executor.shutdown();
    for (Future<?> future : futures) {
        future.get();
    }
}