Examples with ExecutorService java.util.concurrent.ExecutorService used on opensource projects

Example 46 with ExecutorService

use of java.util.concurrent.ExecutorService in project pinot by linkedin.

the class UploadRefreshDeleteIntegrationTest method testUploadRefreshDelete.

@Test(enabled = false, dataProvider = "configProvider")
public void testUploadRefreshDelete(String tableName, SegmentVersion version) throws Exception {
    final int THREAD_COUNT = 1;
    final int SEGMENT_COUNT = 5;
    final int MIN_ROWS_PER_SEGMENT = 500;
    final int MAX_ROWS_PER_SEGMENT = 1000;
    final int OPERATIONS_PER_ITERATION = 10;
    final int ITERATION_COUNT = 5;
    final double UPLOAD_PROBABILITY = 0.8d;
    final String[] segmentNames = new String[SEGMENT_COUNT];
    final int[] segmentRowCounts = new int[SEGMENT_COUNT];
    for (int i = 0; i < SEGMENT_COUNT; i++) {
        segmentNames[i] = "segment_" + i;
        segmentRowCounts[i] = 0;
    }
    for (int i = 0; i < ITERATION_COUNT; i++) {
        // Create THREAD_COUNT threads
        ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT);
        // Submit OPERATIONS_PER_ITERATION uploads/deletes
        for (int j = 0; j < OPERATIONS_PER_ITERATION; j++) {
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        ThreadLocalRandom random = ThreadLocalRandom.current();
                        // Pick a random segment
                        int segmentIndex = random.nextInt(SEGMENT_COUNT);
                        String segmentName = segmentNames[segmentIndex];
                        // Pick a random operation
                        if (random.nextDouble() < UPLOAD_PROBABILITY) {
                            // Upload this segment
                            LOGGER.info("Will upload segment {}", segmentName);
                            synchronized (segmentName) {
                                // Create a segment with a random number of rows
                                int segmentRowCount = random.nextInt(MIN_ROWS_PER_SEGMENT, MAX_ROWS_PER_SEGMENT);
                                LOGGER.info("Generating and uploading segment {} with {} rows", segmentName, segmentRowCount);
                                generateAndUploadRandomSegment(segmentName, segmentRowCount);
                                // Store the number of rows
                                LOGGER.info("Uploaded segment {} with {} rows", segmentName, segmentRowCount);
                                segmentRowCounts[segmentIndex] = segmentRowCount;
                            }
                        } else {
                            // Delete this segment
                            LOGGER.info("Will delete segment {}", segmentName);
                            synchronized (segmentName) {
                                // Delete this segment
                                LOGGER.info("Deleting segment {}", segmentName);
                                String reply = sendDeleteRequest(ControllerRequestURLBuilder.baseUrl(CONTROLLER_BASE_API_URL).forSegmentDelete("myresource", segmentName));
                                LOGGER.info("Deletion returned {}", reply);
                                // Set the number of rows to zero
                                LOGGER.info("Deleted segment {}", segmentName);
                                segmentRowCounts[segmentIndex] = 0;
                            }
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });
        }
        // Await for all tasks to complete
        executorService.shutdown();
        executorService.awaitTermination(5L, TimeUnit.MINUTES);
        // Count number of expected rows
        int expectedRowCount = 0;
        for (int segmentRowCount : segmentRowCounts) {
            expectedRowCount += segmentRowCount;
        }
        // Wait for up to one minute for the row count to match the expected row count
        LOGGER.info("Awaiting for the row count to match {}", expectedRowCount);
        int pinotRowCount = (int) getCurrentServingNumDocs(this.tableName);
        long timeInOneMinute = System.currentTimeMillis() + 60 * 1000L;
        while (System.currentTimeMillis() < timeInOneMinute && pinotRowCount != expectedRowCount) {
            LOGGER.info("Row count is {}, expected {}, awaiting for row count to match", pinotRowCount, expectedRowCount);
            Thread.sleep(5000L);
            try {
                pinotRowCount = (int) getCurrentServingNumDocs(this.tableName);
            } catch (Exception e) {
                LOGGER.warn("Caught exception while sending query to Pinot, retrying", e);
            }
        }
        // Compare row counts
        Assert.assertEquals(pinotRowCount, expectedRowCount, "Expected and actual row counts don't match after waiting one minute");
    }
}

Example 47 with ExecutorService

use of java.util.concurrent.ExecutorService in project morphia by mongodb.

the class TestAsListPerf method morphiaQueryingMultithreaded.

@Test
public void morphiaQueryingMultithreaded() throws InterruptedException {
    final Result morphiaQueryThreadsResult = new Result(nbOfTasks);
    final List<MorphiaQueryThread> morphiaThreads = new ArrayList<MorphiaQueryThread>(nbOfTasks);
    for (int i = 0; i < nbOfTasks; i++) {
        morphiaThreads.add(new MorphiaQueryThread(morphiaQueryThreadsResult, nbOfAddresses));
    }
    final ExecutorService morphiaPool = Executors.newFixedThreadPool(threadPool);
    for (final MorphiaQueryThread thread : morphiaThreads) {
        morphiaPool.execute(thread);
    }
    morphiaPool.shutdown();
    morphiaPool.awaitTermination(30, TimeUnit.SECONDS);
    LOG.debug(format("morphiaQueryingMultithreaded - (%d queries) morphia: %4.2f ms/pojo", morphiaQueryThreadsResult.results.size(), morphiaQueryThreadsResult.getAverageTime()));
}

Example 48 with ExecutorService

use of java.util.concurrent.ExecutorService in project jersey by jersey.

the class CompletionStageAgentResource method recommended.

private CompletionStage<List<Recommendation>> recommended(final WebTarget destinationTarget, final ExecutorService executor, final Queue<String> errors) {
    // Recommended places.
    final CompletionStage<List<Destination>> recommended = destinationTarget.path("recommended").request().header("Rx-User", "CompletionStage").rx(executor).get(new GenericType<List<Destination>>() {
    }).exceptionally(throwable -> {
        errors.offer("Recommended: " + throwable.getMessage());
        return Collections.emptyList();
    });
    return recommended.thenCompose(destinations -> {
        final WebTarget finalForecast = forecastTarget;
        final WebTarget finalCalculation = calculationTarget;
        List<CompletionStage<Recommendation>> recommendations = destinations.stream().map(destination -> {
            final CompletionStage<Forecast> forecast = finalForecast.resolveTemplate("destination", destination.getDestination()).request().rx(executor).get(Forecast.class).exceptionally(throwable -> {
                errors.offer("Forecast: " + throwable.getMessage());
                return new Forecast(destination.getDestination(), "N/A");
            });
            final CompletionStage<Calculation> calculation = finalCalculation.resolveTemplate("from", "Moon").resolveTemplate("to", destination.getDestination()).request().rx(executor).get(Calculation.class).exceptionally(throwable -> {
                errors.offer("Calculation: " + throwable.getMessage());
                return new Calculation("Moon", destination.getDestination(), -1);
            });
            return CompletableFuture.completedFuture(new Recommendation(destination)).thenCombine(forecast, Recommendation::forecast).thenCombine(calculation, Recommendation::calculation);
        }).collect(Collectors.toList());
        return sequence(recommendations);
    });
}

Example 49 with ExecutorService

use of java.util.concurrent.ExecutorService in project jersey by jersey.

the class ManagedAsyncResourceTest method testChatResource.

@Test
public void testChatResource() throws InterruptedException {
    final WebTarget resourceTarget = target().path("chat");
    final int MAX_MESSAGES = 100;
    final int LATCH_WAIT_TIMEOUT = 10 * getAsyncTimeoutMultiplier();
    final boolean debugMode = false;
    final boolean sequentialGet = false;
    final boolean sequentialPost = false;
    final Object sequentialGetLock = new Object();
    final Object sequentialPostLock = new Object();
    final ExecutorService executor = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("async-resource-test-%d").setUncaughtExceptionHandler(new JerseyProcessingUncaughtExceptionHandler()).build());
    final Map<Integer, Integer> postResponses = new ConcurrentHashMap<Integer, Integer>();
    final Map<Integer, Message> getResponses = new ConcurrentHashMap<Integer, Message>();
    final CountDownLatch postRequestLatch = new CountDownLatch(MAX_MESSAGES);
    final CountDownLatch getRequestLatch = new CountDownLatch(MAX_MESSAGES);
    try {
        for (int i = 0; i < MAX_MESSAGES; i++) {
            final int requestId = i;
            executor.submit(new Runnable() {

                @Override
                public void run() {
                    if (debugMode || sequentialPost) {
                        synchronized (sequentialPostLock) {
                            post();
                        }
                    } else {
                        post();
                    }
                }

                private void post() {
                    try {
                        int attemptCounter = 0;
                        while (true) {
                            attemptCounter++;
                            try {
                                final Response response = resourceTarget.request().post(Entity.json(new Message("" + requestId, "" + requestId)));
                                postResponses.put(requestId, response.getStatus());
                                break;
                            } catch (Throwable t) {
                                LOGGER.log(Level.WARNING, String.format("Error POSTING message <%s> for %d. time.", requestId, attemptCounter), t);
                            }
                            if (attemptCounter > 3) {
                                break;
                            }
                            Thread.sleep(10);
                        }
                    } catch (InterruptedException ignored) {
                        LOGGER.log(Level.WARNING, String.format("Error POSTING message <%s>: Interrupted", requestId), ignored);
                    } finally {
                        postRequestLatch.countDown();
                    }
                }
            });
            executor.submit(new Runnable() {

                @Override
                public void run() {
                    if (debugMode || sequentialGet) {
                        synchronized (sequentialGetLock) {
                            get();
                        }
                    } else {
                        get();
                    }
                }

                private void get() {
                    try {
                        int attemptCounter = 0;
                        while (true) {
                            attemptCounter++;
                            try {
                                final Message response = resourceTarget.request("application/json").get(Message.class);
                                getResponses.put(requestId, response);
                                break;
                            } catch (Throwable t) {
                                LOGGER.log(Level.SEVERE, String.format("Error sending GET request <%s> for %d. time.", requestId, attemptCounter), t);
                            }
                            if (attemptCounter > 3) {
                                break;
                            }
                            Thread.sleep(10);
                        }
                    } catch (InterruptedException ignored) {
                        LOGGER.log(Level.WARNING, String.format("Error sending GET message <%s>: Interrupted", requestId), ignored);
                    } finally {
                        getRequestLatch.countDown();
                    }
                }
            });
        }
        if (debugMode) {
            postRequestLatch.await();
            getRequestLatch.await();
        } else {
            if (!postRequestLatch.await(LATCH_WAIT_TIMEOUT, TimeUnit.SECONDS)) {
                LOGGER.log(Level.SEVERE, "Waiting for all POST requests to complete has timed out.");
            }
            if (!getRequestLatch.await(LATCH_WAIT_TIMEOUT, TimeUnit.SECONDS)) {
                LOGGER.log(Level.SEVERE, "Waiting for all GET requests to complete has timed out.");
            }
        }
    } finally {
        executor.shutdownNow();
    }
    StringBuilder messageBuilder = new StringBuilder("POST responses received: ").append(postResponses.size()).append("\n");
    for (Map.Entry<Integer, Integer> postResponseEntry : postResponses.entrySet()) {
        messageBuilder.append("POST response for message ").append(postResponseEntry.getKey()).append(": ").append(postResponseEntry.getValue()).append('\n');
    }
    messageBuilder.append('\n');
    messageBuilder.append("GET responses received: ").append(getResponses.size()).append("\n");
    for (Map.Entry<Integer, Message> getResponseEntry : getResponses.entrySet()) {
        messageBuilder.append("GET response for message ").append(getResponseEntry.getKey()).append(": ").append(getResponseEntry.getValue()).append('\n');
    }
    LOGGER.info(messageBuilder.toString());
    for (Map.Entry<Integer, Integer> postResponseEntry : postResponses.entrySet()) {
        assertEquals("Unexpected POST notification response for message " + postResponseEntry.getKey(), 200, postResponseEntry.getValue().intValue());
    }
    final List<Integer> lost = new LinkedList<Integer>();
    final Collection<Message> getResponseValues = getResponses.values();
    for (int i = 0; i < MAX_MESSAGES; i++) {
        if (!getResponseValues.contains(new Message("" + i, "" + i))) {
            lost.add(i);
        }
    }
    if (!lost.isEmpty()) {
        fail("Detected a posted message loss(es): " + lost.toString());
    }
    assertEquals(MAX_MESSAGES, postResponses.size());
    assertEquals(MAX_MESSAGES, getResponses.size());
}

Example 50 with ExecutorService

use of java.util.concurrent.ExecutorService in project jersey by jersey.

the class ManagedAsyncResourceTest method testLongRunningResource.

@Test
public void testLongRunningResource() throws InterruptedException {
    final WebTarget resourceTarget = target().path(App.ASYNC_LONG_RUNNING_MANAGED_OP_PATH);
    final String expectedResponse = SimpleJerseyExecutorManagedLongRunningResource.NOTIFICATION_RESPONSE;
    final int MAX_MESSAGES = 100;
    final int LATCH_WAIT_TIMEOUT = 10 * getAsyncTimeoutMultiplier();
    final boolean debugMode = false;
    final boolean sequentialGet = false;
    final Object sequentialGetLock = new Object();
    final ExecutorService executor = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("async-resource-test-%d").setUncaughtExceptionHandler(new JerseyProcessingUncaughtExceptionHandler()).build());
    final Map<Integer, String> getResponses = new ConcurrentHashMap<Integer, String>();
    final CountDownLatch getRequestLatch = new CountDownLatch(MAX_MESSAGES);
    try {
        for (int i = 0; i < MAX_MESSAGES; i++) {
            final int requestId = i;
            executor.submit(new Runnable() {

                @Override
                public void run() {
                    if (debugMode || sequentialGet) {
                        synchronized (sequentialGetLock) {
                            get();
                        }
                    } else {
                        get();
                    }
                }

                private void get() {
                    try {
                        int attemptCounter = 0;
                        while (true) {
                            attemptCounter++;
                            try {
                                final String response = resourceTarget.queryParam("id", requestId).request().get(String.class);
                                getResponses.put(requestId, response);
                                break;
                            } catch (Throwable t) {
                                LOGGER.log(Level.SEVERE, String.format("Error sending GET request <%s> for %d. time.", requestId, attemptCounter), t);
                            }
                            if (attemptCounter > 3) {
                                break;
                            }
                            Thread.sleep(10);
                        }
                    } catch (InterruptedException ignored) {
                        LOGGER.log(Level.WARNING, String.format("Error sending GET message <%s>: Interrupted", requestId), ignored);
                    } finally {
                        getRequestLatch.countDown();
                    }
                }
            });
        }
        if (debugMode) {
            getRequestLatch.await();
        } else {
            if (!getRequestLatch.await(LATCH_WAIT_TIMEOUT, TimeUnit.SECONDS)) {
                LOGGER.log(Level.SEVERE, "Waiting for all GET requests to complete has timed out.");
            }
        }
    } finally {
        executor.shutdownNow();
    }
    StringBuilder messageBuilder = new StringBuilder("GET responses received: ").append(getResponses.size()).append("\n");
    for (Map.Entry<Integer, String> getResponseEntry : getResponses.entrySet()) {
        messageBuilder.append("GET response for message ").append(getResponseEntry.getKey()).append(": ").append(getResponseEntry.getValue()).append('\n');
    }
    LOGGER.info(messageBuilder.toString());
    for (Map.Entry<Integer, String> entry : getResponses.entrySet()) {
        assertTrue("Unexpected GET notification response for message " + entry.getKey(), entry.getValue().contains(expectedResponse));
    }
    assertEquals(MAX_MESSAGES, getResponses.size());
}