Examples with Event com.couchbase.connector.dcp.Event used on opensource projects

Example 1 with Event

use of com.couchbase.connector.dcp.Event in project couchbase-elasticsearch-connector by couchbase.

the class ElasticsearchWriter method flush.

public void flush() throws InterruptedException {
    if (buffer.isEmpty()) {
        return;
    }
    try {
        synchronized (this) {
            requestInProgress = true;
            requestStartNanos = System.nanoTime();
        }
        final int totalActionCount = buffer.size();
        final int totalEstimatedBytes = bufferBytes;
        LOGGER.debug("Starting bulk request: {} actions for ~{} bytes", totalActionCount, totalEstimatedBytes);
        final long startNanos = System.nanoTime();
        List<EventDocWriteRequest> requests = new ArrayList<>(buffer.values());
        clearBuffer();
        final Iterator<TimeValue> waitIntervals = backoffPolicy.iterator();
        final Map<Integer, EventDocWriteRequest> vbucketToLastEvent = lenientIndex(r -> r.getEvent().getVbucket(), requests);
        int attemptCounter = 1;
        long indexingTookNanos = 0;
        long totalRetryDelayMillis = 0;
        while (true) {
            if (Thread.interrupted()) {
                requests.forEach(r -> r.getEvent().release());
                Thread.currentThread().interrupt();
                return;
            }
            DocumentLifecycle.logEsWriteStarted(requests, attemptCounter);
            if (attemptCounter == 1) {
                LOGGER.debug("Bulk request attempt #{}", attemptCounter++);
            } else {
                LOGGER.info("Bulk request attempt #{}", attemptCounter++);
            }
            final List<EventDocWriteRequest> requestsToRetry = new ArrayList<>(0);
            final BulkRequest bulkRequest = newBulkRequest(requests);
            bulkRequest.timeout(bulkRequestTimeout);
            final RetryReporter retryReporter = RetryReporter.forLogger(LOGGER);
            try {
                final BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
                final long nowNanos = System.nanoTime();
                final BulkItemResponse[] responses = bulkResponse.getItems();
                indexingTookNanos += bulkResponse.getTook().nanos();
                for (int i = 0; i < responses.length; i++) {
                    final BulkItemResponse response = responses[i];
                    final BulkItemResponse.Failure failure = ElasticsearchHelper.getFailure(response);
                    final EventDocWriteRequest request = requests.get(i);
                    final Event e = request.getEvent();
                    if (failure == null) {
                        updateLatencyMetrics(e, nowNanos);
                        DocumentLifecycle.logEsWriteSucceeded(request);
                        e.release();
                        continue;
                    }
                    if (isRetryable(failure)) {
                        retryReporter.add(e, failure);
                        requestsToRetry.add(request);
                        DocumentLifecycle.logEsWriteFailedWillRetry(request);
                        continue;
                    }
                    if (request instanceof EventRejectionIndexRequest) {
                        // ES rejected the rejection log entry! Total fail.
                        LOGGER.error("Failed to index rejection document for event {}; status code: {} {}", redactUser(e), failure.getStatus(), failure.getMessage());
                        Metrics.rejectionLogFailureCounter().increment();
                        updateLatencyMetrics(e, nowNanos);
                        e.release();
                    } else {
                        LOGGER.warn("Permanent failure to index event {}; status code: {} {}", redactUser(e), failure.getStatus(), failure.getMessage());
                        Metrics.rejectionCounter().increment();
                        DocumentLifecycle.logEsWriteRejected(request, failure.getStatus().getStatus(), failure.getMessage());
                        // don't release event; the request factory assumes ownership
                        final EventRejectionIndexRequest rejectionLogRequest = requestFactory.newRejectionLogRequest(request, failure);
                        if (rejectionLogRequest != null) {
                            requestsToRetry.add(rejectionLogRequest);
                        }
                    }
                    runQuietly("error listener", () -> errorListener.onFailedIndexResponse(e, response));
                }
                Metrics.indexingRetryCounter().increment(requestsToRetry.size());
                requests = requestsToRetry;
            } catch (ElasticsearchStatusException e) {
                if (e.status() == RestStatus.UNAUTHORIZED) {
                    LOGGER.warn("Elasticsearch credentials no longer valid.");
                // todo coordinator.awaitNewConfig("Elasticsearch credentials no longer valid.")
                }
                // Anything else probably means the cluster topology is in transition. Retry!
                LOGGER.warn("Bulk request failed with status {}", e.status(), e);
            } catch (IOException e) {
                // In all of these cases, retry the request!
                if (ThrowableHelper.hasCause(e, ConnectException.class)) {
                    LOGGER.debug("Elasticsearch connect exception", e);
                    LOGGER.warn("Bulk request failed; could not connect to Elasticsearch.");
                } else {
                    LOGGER.warn("Bulk request failed", e);
                }
            } catch (RuntimeException e) {
                requests.forEach(r -> r.getEvent().release());
                // If the worker thread was interrupted, someone wants the worker to stop!
                propagateCauseIfPossible(e, InterruptedException.class);
                // todo retry a few times instead of throwing???
                throw e;
            }
            if (requests.isEmpty()) {
                // EXIT!
                for (Map.Entry<Integer, EventDocWriteRequest> entry : vbucketToLastEvent.entrySet()) {
                    final int vbucket = entry.getKey();
                    Checkpoint checkpoint = entry.getValue().getEvent().getCheckpoint();
                    checkpoint = adjustForIgnoredEvents(vbucket, checkpoint);
                    checkpointService.set(entry.getKey(), checkpoint);
                }
                // were no writes for the same vbucket
                for (Map.Entry<Integer, Checkpoint> entry : ignoreBuffer.entrySet()) {
                    checkpointService.set(entry.getKey(), entry.getValue());
                }
                Metrics.bytesCounter().increment(totalEstimatedBytes);
                Metrics.indexTimePerDocument().record(indexingTookNanos / totalActionCount, NANOSECONDS);
                if (totalRetryDelayMillis != 0) {
                    Metrics.retryDelayTimer().record(totalRetryDelayMillis, MILLISECONDS);
                }
                if (LOGGER.isInfoEnabled()) {
                    final long elapsedMillis = NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                    final ByteSizeValue prettySize = new ByteSizeValue(totalEstimatedBytes, ByteSizeUnit.BYTES);
                    LOGGER.info("Wrote {} actions ~{} in {} ms", totalActionCount, prettySize, elapsedMillis);
                }
                return;
            }
            // retry!
            retryReporter.report();
            Metrics.bulkRetriesCounter().increment();
            // todo check for hasNext? bail out or continue?
            final TimeValue retryDelay = waitIntervals.next();
            LOGGER.info("Retrying bulk request in {}", retryDelay);
            MILLISECONDS.sleep(retryDelay.millis());
            totalRetryDelayMillis += retryDelay.millis();
        }
    } finally {
        synchronized (this) {
            requestInProgress = false;
        }
    }
}