Examples with ThreadPoolExecutor java.util.concurrent.ThreadPoolExecutor used on opensource projects

Example 66 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hbase by apache.

the class RegionReplicaReplicationEndpoint method getDefaultThreadPool.

/**
   * Returns a Thread pool for the RPC's to region replicas. Similar to
   * Connection's thread pool.
   */
private ExecutorService getDefaultThreadPool(Configuration conf) {
    int maxThreads = conf.getInt("hbase.region.replica.replication.threads.max", 256);
    if (maxThreads == 0) {
        maxThreads = Runtime.getRuntime().availableProcessors() * 8;
    }
    long keepAliveTime = conf.getLong("hbase.region.replica.replication.threads.keepalivetime", 60);
    LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(maxThreads * conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
    ThreadPoolExecutor tpe = new ThreadPoolExecutor(maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory(this.getClass().getSimpleName() + "-rpc-shared-"));
    tpe.allowCoreThreadTimeOut(true);
    return tpe;
}

Example 67 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hbase by apache.

the class RegionServerSnapshotManager method initialize.

/**
   * Create a default snapshot handler - uses a zookeeper based member controller.
   * @param rss region server running the handler
   * @throws KeeperException if the zookeeper cluster cannot be reached
   */
@Override
public void initialize(RegionServerServices rss) throws KeeperException {
    this.rss = rss;
    ZooKeeperWatcher zkw = rss.getZooKeeper();
    this.memberRpcs = new ZKProcedureMemberRpcs(zkw, SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION);
    // read in the snapshot request configuration properties
    Configuration conf = rss.getConfiguration();
    long keepAlive = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
    int opThreads = conf.getInt(SNAPSHOT_REQUEST_THREADS_KEY, SNAPSHOT_REQUEST_THREADS_DEFAULT);
    // create the actual snapshot procedure member
    ThreadPoolExecutor pool = ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive);
    this.member = new ProcedureMember(memberRpcs, pool, new SnapshotSubprocedureBuilder());
}

Example 68 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hbase by apache.

the class CompactSplitThread method requestCompactionInternal.

/**
   * @param r region store belongs to
   * @param s Store to request compaction on
   * @param why Why compaction requested -- used in debug messages
   * @param priority override the default priority (NO_PRIORITY == decide)
   * @param request custom compaction request. Can be <tt>null</tt> in which case a simple
   *          compaction will be used.
   */
private synchronized CompactionRequest requestCompactionInternal(final Region r, final Store s, final String why, int priority, CompactionRequest request, boolean selectNow, User user) throws IOException {
    if (this.server.isStopped() || (r.getTableDesc() != null && !r.getTableDesc().isCompactionEnabled())) {
        return null;
    }
    CompactionContext compaction = null;
    if (selectNow) {
        compaction = selectCompaction(r, s, priority, request, user);
        // message logged inside
        if (compaction == null)
            return null;
    }
    // We assume that most compactions are small. So, put system compactions into small
    // pool; we will do selection there, and move to large pool if necessary.
    ThreadPoolExecutor pool = (selectNow && s.throttleCompaction(compaction.getRequest().getSize())) ? longCompactions : shortCompactions;
    pool.execute(new CompactionRunner(s, r, compaction, pool, user));
    if (LOG.isDebugEnabled()) {
        String type = (pool == shortCompactions) ? "Small " : "Large ";
        LOG.debug(type + "Compaction requested: " + (selectNow ? compaction.toString() : "system") + (why != null && !why.isEmpty() ? "; Because: " + why : "") + "; " + this);
    }
    return selectNow ? compaction.getRequest() : null;
}

Example 69 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hbase by apache.

the class DisabledTableSnapshotHandler method snapshotRegions.

// TODO consider parallelizing these operations since they are independent. Right now its just
// easier to keep them serial though
@Override
public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException, KeeperException {
    try {
        // 1. get all the regions hosting this table.
        // extract each pair to separate lists
        Set<HRegionInfo> regions = new HashSet<>();
        for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
            // Don't include non-default regions
            HRegionInfo hri = p.getFirst();
            if (RegionReplicaUtil.isDefaultReplica(hri)) {
                regions.add(hri);
            }
        }
        // handle the mob files if any.
        boolean mobEnabled = MobUtils.hasMobColumns(htd);
        if (mobEnabled) {
            // snapshot the mob files as a offline region.
            HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(htd.getTableName());
            regions.add(mobRegionInfo);
        }
        // 2. for each region, write all the info to disk
        String msg = "Starting to write region info and WALs for regions for offline snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot);
        LOG.info(msg);
        status.setStatus(msg);
        ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "DisabledTableSnapshot");
        try {
            ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {

                @Override
                public void editRegion(final HRegionInfo regionInfo) throws IOException {
                    snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo);
                }
            });
        } finally {
            exec.shutdown();
        }
    } catch (Exception e) {
        // make sure we capture the exception to propagate back to the client later
        String reason = "Failed snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " due to exception:" + e.getMessage();
        ForeignException ee = new ForeignException(reason, e);
        monitor.receive(ee);
        status.abort("Snapshot of table: " + snapshotTable + " failed because " + e.getMessage());
    } finally {
        LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot) + " as finished.");
    }
}

Example 70 with ThreadPoolExecutor

use of java.util.concurrent.ThreadPoolExecutor in project hbase by apache.

the class MasterFlushTableProcedureManager method initialize.

@Override
public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException, IOException, UnsupportedOperationException {
    this.master = master;
    // get the configuration for the coordinator
    Configuration conf = master.getConfiguration();
    long wakeFrequency = conf.getInt(FLUSH_WAKE_MILLIS_KEY, FLUSH_WAKE_MILLIS_DEFAULT);
    long timeoutMillis = conf.getLong(FLUSH_TIMEOUT_MILLIS_KEY, FLUSH_TIMEOUT_MILLIS_DEFAULT);
    int threads = conf.getInt(FLUSH_PROC_POOL_THREADS_KEY, FLUSH_PROC_POOL_THREADS_DEFAULT);
    // setup the procedure coordinator
    String name = master.getServerName().toString();
    ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, threads);
    ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinator(master.getZooKeeper(), getProcedureSignature(), name);
    this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
}