Examples with BulkByScrollResponse org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse used on opensource projects

Example 6 with BulkByScrollResponse

use of org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse in project elasticsearch by elastic.

the class RethrottleTests method testCase.

private void testCase(AbstractBulkByScrollRequestBuilder<?, ?> request, String actionName) throws Exception {
    logger.info("Starting test for [{}] with [{}] slices", actionName, request.request().getSlices());
    /* Add ten documents per slice so most slices will have many documents to process, having to go to multiple batches.
         * we can't rely on all of them doing so, but 
         */
    List<IndexRequestBuilder> docs = new ArrayList<>();
    for (int i = 0; i < request.request().getSlices() * 10; i++) {
        docs.add(client().prepareIndex("test", "test", Integer.toString(i)).setSource("foo", "bar"));
    }
    indexRandom(true, docs);
    // Start a request that will never finish unless we rethrottle it
    // Throttle "forever"
    request.setRequestsPerSecond(.000001f);
    // Make sure we use multiple batches
    request.source().setSize(1);
    ListenableActionFuture<? extends BulkByScrollResponse> responseListener = request.execute();
    TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, request.request().getSlices());
    TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId();
    if (request.request().getSlices() == 1) {
        assertThat(taskGroupToRethrottle.getChildTasks(), empty());
    } else {
        // There should be a sane number of child tasks running
        assertThat(taskGroupToRethrottle.getChildTasks(), hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(request.request().getSlices()))));
        // Wait for all of the sub tasks to start (or finish, some might finish early, all that matters is that not all do)
        assertBusy(() -> {
            BulkByScrollTask.Status parent = (BulkByScrollTask.Status) client().admin().cluster().prepareGetTask(taskToRethrottle).get().getTask().getTask().getStatus();
            long finishedSubTasks = parent.getSliceStatuses().stream().filter(s -> s != null).count();
            ListTasksResponse list = client().admin().cluster().prepareListTasks().setParentTaskId(taskToRethrottle).get();
            list.rethrowFailures("subtasks");
            assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) request.request().getSlices()));
            assertThat(list.getTasks().size(), greaterThan(0));
        });
    }
    // Now rethrottle it so it'll finish
    // No throttle or "very fast"
    float newRequestsPerSecond = randomBoolean() ? Float.POSITIVE_INFINITY : between(1, 1000) * 100000;
    ListTasksResponse rethrottleResponse = rethrottle().setTaskId(taskToRethrottle).setRequestsPerSecond(newRequestsPerSecond).get();
    rethrottleResponse.rethrowFailures("Rethrottle");
    assertThat(rethrottleResponse.getTasks(), hasSize(1));
    BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus();
    // Now check the resulting requests per second.
    if (request.request().getSlices() == 1) {
        // If there is a single slice it should match perfectly
        assertEquals(newRequestsPerSecond, status.getRequestsPerSecond(), Float.MIN_NORMAL);
    } else {
        /* Check that at least one slice was rethrottled. We won't always rethrottle all of them because they might have completed.
             * With multiple slices these numbers might not add up perfectly, thus the 1.01F. */
        long unfinished = status.getSliceStatuses().stream().filter(slice -> slice != null).filter(slice -> slice.getStatus().getTotal() > slice.getStatus().getSuccessfullyProcessed()).count();
        float maxExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / unfinished) * 1.01F;
        float minExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / request.request().getSlices()) * 0.99F;
        boolean oneSliceRethrottled = false;
        float totalRequestsPerSecond = 0;
        for (BulkByScrollTask.StatusOrException statusOrException : status.getSliceStatuses()) {
            if (statusOrException == null) {
                /* The slice can be null here because it was completed but hadn't reported its success back to the task when the
                     * rethrottle request came through. */
                continue;
            }
            assertNull(statusOrException.getException());
            BulkByScrollTask.Status slice = statusOrException.getStatus();
            if (slice.getTotal() > slice.getSuccessfullyProcessed()) {
                // This slice reports as not having completed so it should have been processed.
                assertThat(slice.getRequestsPerSecond(), both(greaterThanOrEqualTo(minExpectedSliceRequestsPerSecond)).and(lessThanOrEqualTo(maxExpectedSliceRequestsPerSecond)));
            }
            if (minExpectedSliceRequestsPerSecond <= slice.getRequestsPerSecond() && slice.getRequestsPerSecond() <= maxExpectedSliceRequestsPerSecond) {
                oneSliceRethrottled = true;
            }
            totalRequestsPerSecond += slice.getRequestsPerSecond();
        }
        assertTrue("At least one slice must be rethrottled", oneSliceRethrottled);
        /* Now assert that the parent request has the total requests per second. This is a much weaker assertion than that the parent
             * actually has the newRequestsPerSecond. For the most part it will. Sometimes it'll be greater because only unfinished requests
             * are rethrottled, the finished ones just keep whatever requests per second they had while they were running. But it might
             * also be less than newRequestsPerSecond because the newRequestsPerSecond is divided among running sub-requests and then the
             * requests are rethrottled. If one request finishes in between the division and the application of the new throttle then it
             * won't be rethrottled, thus only contributing its lower total. */
        assertEquals(totalRequestsPerSecond, status.getRequestsPerSecond(), totalRequestsPerSecond * 0.0001f);
    }
    // Now the response should come back quickly because we've rethrottled the request
    BulkByScrollResponse response = responseListener.get();
    assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.", response.getBatches(), greaterThanOrEqualTo(request.request().getSlices()));
}

Example 7 with BulkByScrollResponse

use of org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse in project elasticsearch by elastic.

the class RetryTests method testCase.

private void testCase(String action, AbstractBulkByScrollRequestBuilder<?, ?> request, BulkIndexByScrollResponseMatcher matcher) throws Exception {
    logger.info("Blocking search");
    CyclicBarrier initialSearchBlock = blockExecutor(ThreadPool.Names.SEARCH);
    // Make sure we use more than one batch so we have to scroll
    request.source().setSize(DOC_COUNT / randomIntBetween(2, 10));
    logger.info("Starting request");
    ListenableActionFuture<BulkByScrollResponse> responseListener = request.execute();
    try {
        logger.info("Waiting for search rejections on the initial search");
        assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(0L)));
        logger.info("Blocking bulk and unblocking search so we start to get bulk rejections");
        CyclicBarrier bulkBlock = blockExecutor(ThreadPool.Names.BULK);
        initialSearchBlock.await();
        logger.info("Waiting for bulk rejections");
        assertBusy(() -> assertThat(taskStatus(action).getBulkRetries(), greaterThan(0L)));
        // Keep a copy of the current number of search rejections so we can assert that we get more when we block the scroll
        long initialSearchRejections = taskStatus(action).getSearchRetries();
        logger.info("Blocking search and unblocking bulk so we should get search rejections for the scroll");
        CyclicBarrier scrollBlock = blockExecutor(ThreadPool.Names.SEARCH);
        bulkBlock.await();
        logger.info("Waiting for search rejections for the scroll");
        assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(initialSearchRejections)));
        logger.info("Unblocking the scroll");
        scrollBlock.await();
        logger.info("Waiting for the request to finish");
        BulkByScrollResponse response = responseListener.get();
        assertThat(response, matcher);
        assertThat(response.getBulkRetries(), greaterThan(0L));
        assertThat(response.getSearchRetries(), greaterThan(initialSearchRejections));
    } finally {
        // Fetch the response just in case we blew up half way through. This will make sure the failure is thrown up to the top level.
        BulkByScrollResponse response = responseListener.get();
        assertThat(response.getSearchFailures(), empty());
        assertThat(response.getBulkFailures(), empty());
    }
}

Example 8 with BulkByScrollResponse

use of org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse in project elasticsearch by elastic.

the class TransportRethrottleActionTests method testRethrottleWithSomeSucceeded.

public void testRethrottleWithSomeSucceeded() {
    int succeeded = between(1, slices - 1);
    List<BulkByScrollTask.StatusOrException> sliceStatuses = new ArrayList<>(slices);
    for (int i = 0; i < succeeded; i++) {
        BulkByScrollTask.Status status = believeableCompletedStatus(i);
        task.onSliceResponse(neverCalled(), i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false));
        sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
    }
    List<TaskInfo> tasks = new ArrayList<>();
    for (int i = succeeded; i < slices; i++) {
        BulkByScrollTask.Status status = believeableInProgressStatus(i);
        tasks.add(new TaskInfo(new TaskId("test", 123), "test", "test", "test", status, 0, 0, true, new TaskId("test", task.getId())));
        sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
    }
    rethrottleTestCase(slices - succeeded, listener -> listener.onResponse(new ListTasksResponse(tasks, emptyList(), emptyList())), expectSuccessfulRethrottleWithStatuses(sliceStatuses));
}

Example 9 with BulkByScrollResponse

use of org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse in project elasticsearch by elastic.

the class UpdateByQueryWhileModifyingTests method testUpdateWhileReindexing.

public void testUpdateWhileReindexing() throws Exception {
    AtomicReference<String> value = new AtomicReference<>(randomSimpleString(random()));
    indexRandom(true, client().prepareIndex("test", "test", "test").setSource("test", value.get()));
    AtomicReference<Exception> failure = new AtomicReference<>();
    AtomicBoolean keepUpdating = new AtomicBoolean(true);
    Thread updater = new Thread(() -> {
        while (keepUpdating.get()) {
            try {
                BulkByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
                assertThat(response, matcher().updated(either(equalTo(0L)).or(equalTo(1L))).versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
            } catch (Exception e) {
                failure.set(e);
            }
        }
    });
    updater.start();
    try {
        for (int i = 0; i < MAX_MUTATIONS; i++) {
            GetResponse get = client().prepareGet("test", "test", "test").get();
            assertEquals(value.get(), get.getSource().get("test"));
            value.set(randomSimpleString(random()));
            IndexRequestBuilder index = client().prepareIndex("test", "test", "test").setSource("test", value.get()).setRefreshPolicy(IMMEDIATE);
            /*
                 * Update by query increments the version number so concurrent
                 * indexes might get version conflict exceptions so we just
                 * blindly retry.
                 */
            int attempts = 0;
            while (true) {
                attempts++;
                try {
                    index.setVersion(get.getVersion()).get();
                    break;
                } catch (VersionConflictEngineException e) {
                    if (attempts >= MAX_ATTEMPTS) {
                        throw new RuntimeException("Failed to index after [" + MAX_ATTEMPTS + "] attempts. Too many version conflicts!");
                    }
                    logger.info("Caught expected version conflict trying to perform mutation number [{}] with version [{}] " + "on attempt [{}]. Retrying.", i, get.getVersion(), attempts);
                    get = client().prepareGet("test", "test", "test").get();
                }
            }
        }
    } finally {
        keepUpdating.set(false);
        updater.join(TimeUnit.SECONDS.toMillis(10));
        if (failure.get() != null) {
            throw new RuntimeException(failure.get());
        }
    }
}

Example 10 with BulkByScrollResponse

use of org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse in project elasticsearch by elastic.

the class BulkIndexByScrollResponseTests method testMergeConstructor.

public void testMergeConstructor() {
    int mergeCount = between(2, 10);
    List<BulkByScrollResponse> responses = new ArrayList<>(mergeCount);
    int took = between(1000, 10000);
    int tookIndex = between(0, mergeCount - 1);
    List<BulkItemResponse.Failure> allBulkFailures = new ArrayList<>();
    List<SearchFailure> allSearchFailures = new ArrayList<>();
    boolean timedOut = false;
    String reasonCancelled = rarely() ? randomAsciiOfLength(5) : null;
    for (int i = 0; i < mergeCount; i++) {
        // One of the merged responses gets the expected value for took, the others get a smaller value
        TimeValue thisTook = timeValueMillis(i == tookIndex ? took : between(0, took));
        // The actual status doesn't matter too much - we test merging those elsewhere
        String thisReasonCancelled = rarely() ? randomAsciiOfLength(5) : null;
        BulkByScrollTask.Status status = new BulkByScrollTask.Status(i, 0, 0, 0, 0, 0, 0, 0, 0, 0, timeValueMillis(0), 0f, thisReasonCancelled, timeValueMillis(0));
        List<BulkItemResponse.Failure> bulkFailures = frequently() ? emptyList() : IntStream.range(0, between(1, 3)).mapToObj(j -> new BulkItemResponse.Failure("idx", "type", "id", new Exception())).collect(Collectors.toList());
        allBulkFailures.addAll(bulkFailures);
        List<SearchFailure> searchFailures = frequently() ? emptyList() : IntStream.range(0, between(1, 3)).mapToObj(j -> new SearchFailure(new Exception())).collect(Collectors.toList());
        allSearchFailures.addAll(searchFailures);
        boolean thisTimedOut = rarely();
        timedOut |= thisTimedOut;
        responses.add(new BulkByScrollResponse(thisTook, status, bulkFailures, searchFailures, thisTimedOut));
    }
    BulkByScrollResponse merged = new BulkByScrollResponse(responses, reasonCancelled);
    assertEquals(timeValueMillis(took), merged.getTook());
    assertEquals(allBulkFailures, merged.getBulkFailures());
    assertEquals(allSearchFailures, merged.getSearchFailures());
    assertEquals(timedOut, merged.isTimedOut());
    assertEquals(reasonCancelled, merged.getReasonCancelled());
}