Examples with RejectedExecutionException java.util.concurrent.RejectedExecutionException used on opensource projects

Example 16 with RejectedExecutionException

use of java.util.concurrent.RejectedExecutionException in project hadoop by apache.

the class TestFileSystemOperationsWithThreads method testDeleteThreadPoolExecuteFailure.

/*
   * Test case for delete operation with multiple threads and flat listing enabled.
   */
@Test
public void testDeleteThreadPoolExecuteFailure() throws Exception {
    // Mock thread pool executor to throw exception for all requests.
    ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
    Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
    // Spy azure file system object and return mocked thread pool
    NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
    String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
    AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
    Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
    // With single iteration, we would have created 7 blobs resulting 7 threads.
    Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete", path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
    validateDeleteFolder(mockFs, "root");
    // Validate from logs that threads are disabled.
    String content = logs.getOutput();
    assertTrue(content.contains("Rejected execution of thread for Delete operation on blob"));
    assertTrue(content.contains("Serializing the Delete operation"));
}

Example 17 with RejectedExecutionException

use of java.util.concurrent.RejectedExecutionException in project hive by apache.

the class SQLOperation method runInternal.

@Override
public void runInternal() throws HiveSQLException {
    setState(OperationState.PENDING);
    boolean runAsync = shouldRunAsync();
    final boolean asyncPrepare = runAsync && HiveConf.getBoolVar(queryState.getConf(), HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_ASYNC_COMPILE);
    if (!asyncPrepare) {
        prepare(queryState);
    }
    if (!runAsync) {
        runQuery();
    } else {
        // We'll pass ThreadLocals in the background thread from the foreground (handler) thread.
        // 1) ThreadLocal Hive object needs to be set in background thread
        // 2) The metastore client in Hive is associated with right user.
        // 3) Current UGI will get used by metastore when metastore is in embedded mode
        Runnable work = new BackgroundWork(getCurrentUGI(), parentSession.getSessionHive(), SessionState.getPerfLogger(), SessionState.get(), asyncPrepare);
        try {
            // This submit blocks if no background threads are available to run this operation
            Future<?> backgroundHandle = getParentSession().submitBackgroundOperation(work);
            setBackgroundHandle(backgroundHandle);
        } catch (RejectedExecutionException rejected) {
            setState(OperationState.ERROR);
            throw new HiveSQLException("The background threadpool cannot accept" + " new task for execution, please retry the operation", rejected);
        }
    }
}

Example 18 with RejectedExecutionException

use of java.util.concurrent.RejectedExecutionException in project storm by apache.

the class Localizer method updateBlobs.

/**
   * This function updates blobs on the supervisor. It uses a separate thread pool and runs
   * asynchronously of the download and delete.
   */
public List<LocalizedResource> updateBlobs(List<LocalResource> localResources, String user) throws AuthorizationException, KeyNotFoundException, IOException {
    LocalizedResourceSet lrsrcSet = _userRsrc.get(user);
    ArrayList<LocalizedResource> results = new ArrayList<>();
    ArrayList<Callable<LocalizedResource>> updates = new ArrayList<>();
    if (lrsrcSet == null) {
        // resource set must have been removed
        return results;
    }
    ClientBlobStore blobstore = null;
    try {
        blobstore = getClientBlobStore();
        for (LocalResource localResource : localResources) {
            String key = localResource.getBlobName();
            LocalizedResource lrsrc = lrsrcSet.get(key, localResource.shouldUncompress());
            if (lrsrc == null) {
                LOG.warn("blob requested for update doesn't exist: {}", key);
                continue;
            } else {
                // update it if either the version isn't the latest or if any local blob files are missing
                if (!isLocalizedResourceUpToDate(lrsrc, blobstore) || !isLocalizedResourceDownloaded(lrsrc)) {
                    LOG.debug("updating blob: {}", key);
                    updates.add(new DownloadBlob(this, _conf, key, new File(lrsrc.getFilePath()), user, lrsrc.isUncompressed(), true));
                }
            }
        }
    } finally {
        if (blobstore != null) {
            blobstore.shutdown();
        }
    }
    try {
        List<Future<LocalizedResource>> futures = _updateExecService.invokeAll(updates);
        for (Future<LocalizedResource> futureRsrc : futures) {
            try {
                LocalizedResource lrsrc = futureRsrc.get();
                // put the resource just in case it was removed at same time by the cleaner
                LocalizedResourceSet newSet = new LocalizedResourceSet(user);
                LocalizedResourceSet newlrsrcSet = _userRsrc.putIfAbsent(user, newSet);
                if (newlrsrcSet == null) {
                    newlrsrcSet = newSet;
                }
                newlrsrcSet.putIfAbsent(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
                results.add(lrsrc);
            } catch (ExecutionException e) {
                LOG.error("Error updating blob: ", e);
                if (e.getCause() instanceof AuthorizationException) {
                    throw (AuthorizationException) e.getCause();
                }
                if (e.getCause() instanceof KeyNotFoundException) {
                    throw (KeyNotFoundException) e.getCause();
                }
            }
        }
    } catch (RejectedExecutionException re) {
        LOG.error("Error updating blobs : ", re);
    } catch (InterruptedException ie) {
        throw new IOException("Interrupted Exception", ie);
    }
    return results;
}

Example 19 with RejectedExecutionException

use of java.util.concurrent.RejectedExecutionException in project hive by apache.

the class JobRequestExecutor method execute.

/*
   * Executes job request operation. If thread pool is not created then job request is
   * executed in current thread itself.
   *
   * @param jobExecuteCallable
   *          Callable object to run the job request task.
   *
   */
public T execute(JobCallable<T> jobExecuteCallable) throws InterruptedException, TimeoutException, TooManyRequestsException, ExecutionException {
    /*
     * The callable shouldn't be null to execute. The thread pool also should be configured
     * to execute requests.
     */
    assert (jobExecuteCallable != null);
    assert (this.jobExecutePool != null);
    String type = this.requestType.toString().toLowerCase();
    String retryMessageForConcurrentRequests = "Please wait for some time before retrying " + "the operation. Please refer to the config " + concurrentRequestsConfigName + " to configure concurrent requests.";
    LOG.debug("Starting new " + type + " job request with time out " + this.requestExecutionTimeoutInSec + "seconds.");
    Future<T> future = null;
    try {
        future = this.jobExecutePool.submit(jobExecuteCallable);
    } catch (RejectedExecutionException rejectedException) {
        /*
       * Not able to find thread to execute the job request. Raise Busy exception and client
       * can retry the operation.
       */
        String tooManyRequestsExceptionMessage = "Unable to service the " + type + " job request as " + "templeton service is busy with too many " + type + " job requests. " + retryMessageForConcurrentRequests;
        LOG.warn(tooManyRequestsExceptionMessage);
        throw new TooManyRequestsException(tooManyRequestsExceptionMessage);
    }
    T result = null;
    try {
        result = this.requestExecutionTimeoutInSec > 0 ? future.get(this.requestExecutionTimeoutInSec, TimeUnit.SECONDS) : future.get();
    } catch (TimeoutException e) {
        /*
       * See if the execution thread has just completed operation and result is available.
       * If result is available then return the result. Otherwise, raise exception.
       */
        if ((result = tryGetJobResultOrSetJobStateFailed(jobExecuteCallable)) == null) {
            String message = this.requestType + " job request got timed out. Please wait for some time " + "before retrying the operation. Please refer to the config " + jobTimeoutConfigName + " to configure job request time out.";
            LOG.warn(message);
            /*
         * Throw TimeoutException to caller.
         */
            throw new TimeoutException(message);
        }
    } catch (InterruptedException e) {
        /*
       * See if the execution thread has just completed operation and result is available.
       * If result is available then return the result. Otherwise, raise exception.
       */
        if ((result = tryGetJobResultOrSetJobStateFailed(jobExecuteCallable)) == null) {
            String message = this.requestType + " job request got interrupted. Please wait for some time " + "before retrying the operation.";
            LOG.warn(message);
            /*
         * Throw TimeoutException to caller.
         */
            throw new InterruptedException(message);
        }
    } catch (CancellationException e) {
        /*
       * See if the execution thread has just completed operation and result is available.
       * If result is available then return the result. Otherwise, raise exception.
       */
        if ((result = tryGetJobResultOrSetJobStateFailed(jobExecuteCallable)) == null) {
            String message = this.requestType + " job request got cancelled and thread got interrupted. " + "Please wait for some time before retrying the operation.";
            LOG.warn(message);
            throw new InterruptedException(message);
        }
    } finally {
        /*
       * If the thread is still active and needs to be cancelled then cancel it. This may
       * happen in case task got interrupted, or timed out.
       */
        if (enableCancelTask) {
            cancelExecutePoolThread(future);
        }
    }
    LOG.debug("Completed " + type + " job request.");
    return result;
}

Example 20 with RejectedExecutionException

use of java.util.concurrent.RejectedExecutionException in project hbase by apache.

the class TestPartitionedMobCompactor method createThreadPool.

private static ExecutorService createThreadPool() {
    int maxThreads = 10;
    long keepAliveTime = 60;
    final SynchronousQueue<Runnable> queue = new SynchronousQueue<>();
    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime, TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"), new RejectedExecutionHandler() {

        @Override
        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
            try {
                // waiting for a thread to pick up instead of throwing exceptions.
                queue.put(r);
            } catch (InterruptedException e) {
                throw new RejectedExecutionException(e);
            }
        }
    });
    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
    return pool;
}