Examples with LinkedBlockingQueue java.util.concurrent.LinkedBlockingQueue used on opensource projects

Example 16 with LinkedBlockingQueue

use of java.util.concurrent.LinkedBlockingQueue in project hbase by apache.

the class IntegrationTestSendTraceRequests method insertData.

private LinkedBlockingQueue<Long> insertData() throws IOException, InterruptedException {
    LinkedBlockingQueue<Long> rowKeys = new LinkedBlockingQueue<>(25000);
    BufferedMutator ht = util.getConnection().getBufferedMutator(this.tableName);
    byte[] value = new byte[300];
    for (int x = 0; x < 5000; x++) {
        TraceScope traceScope = Trace.startSpan("insertData", Sampler.ALWAYS);
        try {
            for (int i = 0; i < 5; i++) {
                long rk = random.nextLong();
                rowKeys.add(rk);
                Put p = new Put(Bytes.toBytes(rk));
                for (int y = 0; y < 10; y++) {
                    random.nextBytes(value);
                    p.addColumn(familyName, Bytes.toBytes(random.nextLong()), value);
                }
                ht.mutate(p);
            }
            if ((x % 1000) == 0) {
                admin.flush(tableName);
            }
        } finally {
            traceScope.close();
        }
    }
    admin.flush(tableName);
    return rowKeys;
}

Example 17 with LinkedBlockingQueue

use of java.util.concurrent.LinkedBlockingQueue in project hbase by apache.

the class TestAcidGuarantees method createThreadPool.

private ExecutorService createThreadPool() {
    int maxThreads = 256;
    int coreThreads = 128;
    long keepAliveTime = 60;
    BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<Runnable>(maxThreads * HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS);
    ThreadPoolExecutor tpe = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory(toString() + "-shared"));
    tpe.allowCoreThreadTimeOut(true);
    return tpe;
}

Example 18 with LinkedBlockingQueue

use of java.util.concurrent.LinkedBlockingQueue in project hbase by apache.

the class MultiHConnection method createBatchPool.

// Copied from ConnectionImplementation.getBatchPool()
// We should get rid of this when Connection.processBatchCallback is un-deprecated and provides
// an API to manage a batch pool
private void createBatchPool(Configuration conf) {
    // Use the same config for keep alive as in ConnectionImplementation.getBatchPool();
    int maxThreads = conf.getInt("hbase.multihconnection.threads.max", 256);
    if (maxThreads == 0) {
        maxThreads = Runtime.getRuntime().availableProcessors() * 8;
    }
    long keepAliveTime = conf.getLong("hbase.multihconnection.threads.keepalivetime", 60);
    LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(maxThreads * conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
    ThreadPoolExecutor tpe = new ThreadPoolExecutor(maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory("MultiHConnection" + "-shared-"));
    tpe.allowCoreThreadTimeOut(true);
    this.batchPool = tpe;
}

Example 19 with LinkedBlockingQueue

use of java.util.concurrent.LinkedBlockingQueue in project hive by apache.

the class SessionManager method createBackgroundOperationPool.

private void createBackgroundOperationPool() {
    int poolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS);
    LOG.info("HiveServer2: Background operation thread pool size: " + poolSize);
    int poolQueueSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE);
    LOG.info("HiveServer2: Background operation thread wait queue size: " + poolQueueSize);
    long keepAliveTime = HiveConf.getTimeVar(hiveConf, ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, TimeUnit.SECONDS);
    LOG.info("HiveServer2: Background operation thread keepalive time: " + keepAliveTime + " seconds");
    // Create a thread pool with #poolSize threads
    // Threads terminate when they are idle for more than the keepAliveTime
    // A bounded blocking queue is used to queue incoming operations, if #operations > poolSize
    String threadPoolName = "HiveServer2-Background-Pool";
    final BlockingQueue queue = new LinkedBlockingQueue<Runnable>(poolQueueSize);
    backgroundOperationPool = new ThreadPoolExecutor(poolSize, poolSize, keepAliveTime, TimeUnit.SECONDS, queue, new ThreadFactoryWithGarbageCleanup(threadPoolName));
    backgroundOperationPool.allowCoreThreadTimeOut(true);
    checkInterval = HiveConf.getTimeVar(hiveConf, ConfVars.HIVE_SERVER2_SESSION_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
    sessionTimeout = HiveConf.getTimeVar(hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
    checkOperation = HiveConf.getBoolVar(hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_CHECK_OPERATION);
    Metrics m = MetricsFactory.getInstance();
    if (m != null) {
        m.addGauge(MetricsConstant.EXEC_ASYNC_QUEUE_SIZE, new MetricsVariable() {

            @Override
            public Object getValue() {
                return queue.size();
            }
        });
        m.addGauge(MetricsConstant.EXEC_ASYNC_POOL_SIZE, new MetricsVariable() {

            @Override
            public Object getValue() {
                return backgroundOperationPool.getPoolSize();
            }
        });
    }
}

Example 20 with LinkedBlockingQueue

use of java.util.concurrent.LinkedBlockingQueue in project hive by apache.

the class ATSHook method setupAtsExecutor.

private static void setupAtsExecutor(HiveConf conf) {
    synchronized (LOCK) {
        if (executor == null) {
            // The call to ATS appears to block indefinitely, blocking the ATS thread while
            // the hook continues to submit work to the ExecutorService with each query.
            // Over time the queued items can cause OOM as the HookContext seems to contain
            // some items which use a lot of memory.
            // Prevent this situation by creating executor with bounded capacity -
            // the event will not be sent to ATS if there are too many outstanding work submissions.
            int queueCapacity = conf.getIntVar(HiveConf.ConfVars.ATSHOOKQUEUECAPACITY);
            // Executor to create the ATS events.
            // This can use significant resources and should not be done on the main query thread.
            LOG.info("Creating ATS executor queue with capacity " + queueCapacity);
            BlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>(queueCapacity);
            ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ATS Logger %d").build();
            executor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, queue, threadFactory);
            // Create a separate thread to send the events.
            // Keep separate from the creating events in case the send blocks.
            BlockingQueue<Runnable> senderQueue = new LinkedBlockingQueue<Runnable>(queueCapacity);
            senderExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, senderQueue, threadFactory);
            YarnConfiguration yarnConf = new YarnConfiguration();
            timelineClient = TimelineClient.createTimelineClient();
            timelineClient.init(yarnConf);
            timelineClient.start();
            ShutdownHookManager.addShutdownHook(new Runnable() {

                @Override
                public void run() {
                    try {
                        executor.shutdown();
                        executor.awaitTermination(WAIT_TIME, TimeUnit.SECONDS);
                        executor = null;
                    } catch (InterruptedException ie) {
                    /* ignore */
                    }
                    timelineClient.stop();
                }
            });
        }
    }
}