Examples with Checkpoint com.couchbase.connector.dcp.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of com.couchbase.connector.dcp.Checkpoint in project couchbase-elasticsearch-connector by couchbase.

the class ElasticsearchWriter method flush.

public void flush() throws InterruptedException {
    if (buffer.isEmpty()) {
        return;
    }
    try {
        synchronized (this) {
            requestInProgress = true;
            requestStartNanos = System.nanoTime();
        }
        final int totalActionCount = buffer.size();
        final int totalEstimatedBytes = bufferBytes;
        LOGGER.debug("Starting bulk request: {} actions for ~{} bytes", totalActionCount, totalEstimatedBytes);
        final long startNanos = System.nanoTime();
        List<EventDocWriteRequest> requests = new ArrayList<>(buffer.values());
        clearBuffer();
        final Iterator<TimeValue> waitIntervals = backoffPolicy.iterator();
        final Map<Integer, EventDocWriteRequest> vbucketToLastEvent = lenientIndex(r -> r.getEvent().getVbucket(), requests);
        int attemptCounter = 1;
        long indexingTookNanos = 0;
        long totalRetryDelayMillis = 0;
        while (true) {
            if (Thread.interrupted()) {
                requests.forEach(r -> r.getEvent().release());
                Thread.currentThread().interrupt();
                return;
            }
            DocumentLifecycle.logEsWriteStarted(requests, attemptCounter);
            if (attemptCounter == 1) {
                LOGGER.debug("Bulk request attempt #{}", attemptCounter++);
            } else {
                LOGGER.info("Bulk request attempt #{}", attemptCounter++);
            }
            final List<EventDocWriteRequest> requestsToRetry = new ArrayList<>(0);
            final BulkRequest bulkRequest = newBulkRequest(requests);
            bulkRequest.timeout(bulkRequestTimeout);
            final RetryReporter retryReporter = RetryReporter.forLogger(LOGGER);
            try {
                final BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
                final long nowNanos = System.nanoTime();
                final BulkItemResponse[] responses = bulkResponse.getItems();
                indexingTookNanos += bulkResponse.getTook().nanos();
                for (int i = 0; i < responses.length; i++) {
                    final BulkItemResponse response = responses[i];
                    final BulkItemResponse.Failure failure = ElasticsearchHelper.getFailure(response);
                    final EventDocWriteRequest request = requests.get(i);
                    final Event e = request.getEvent();
                    if (failure == null) {
                        updateLatencyMetrics(e, nowNanos);
                        DocumentLifecycle.logEsWriteSucceeded(request);
                        e.release();
                        continue;
                    }
                    if (isRetryable(failure)) {
                        retryReporter.add(e, failure);
                        requestsToRetry.add(request);
                        DocumentLifecycle.logEsWriteFailedWillRetry(request);
                        continue;
                    }
                    if (request instanceof EventRejectionIndexRequest) {
                        // ES rejected the rejection log entry! Total fail.
                        LOGGER.error("Failed to index rejection document for event {}; status code: {} {}", redactUser(e), failure.getStatus(), failure.getMessage());
                        Metrics.rejectionLogFailureCounter().increment();
                        updateLatencyMetrics(e, nowNanos);
                        e.release();
                    } else {
                        LOGGER.warn("Permanent failure to index event {}; status code: {} {}", redactUser(e), failure.getStatus(), failure.getMessage());
                        Metrics.rejectionCounter().increment();
                        DocumentLifecycle.logEsWriteRejected(request, failure.getStatus().getStatus(), failure.getMessage());
                        // don't release event; the request factory assumes ownership
                        final EventRejectionIndexRequest rejectionLogRequest = requestFactory.newRejectionLogRequest(request, failure);
                        if (rejectionLogRequest != null) {
                            requestsToRetry.add(rejectionLogRequest);
                        }
                    }
                    runQuietly("error listener", () -> errorListener.onFailedIndexResponse(e, response));
                }
                Metrics.indexingRetryCounter().increment(requestsToRetry.size());
                requests = requestsToRetry;
            } catch (ElasticsearchStatusException e) {
                if (e.status() == RestStatus.UNAUTHORIZED) {
                    LOGGER.warn("Elasticsearch credentials no longer valid.");
                // todo coordinator.awaitNewConfig("Elasticsearch credentials no longer valid.")
                }
                // Anything else probably means the cluster topology is in transition. Retry!
                LOGGER.warn("Bulk request failed with status {}", e.status(), e);
            } catch (IOException e) {
                // In all of these cases, retry the request!
                if (ThrowableHelper.hasCause(e, ConnectException.class)) {
                    LOGGER.debug("Elasticsearch connect exception", e);
                    LOGGER.warn("Bulk request failed; could not connect to Elasticsearch.");
                } else {
                    LOGGER.warn("Bulk request failed", e);
                }
            } catch (RuntimeException e) {
                requests.forEach(r -> r.getEvent().release());
                // If the worker thread was interrupted, someone wants the worker to stop!
                propagateCauseIfPossible(e, InterruptedException.class);
                // todo retry a few times instead of throwing???
                throw e;
            }
            if (requests.isEmpty()) {
                // EXIT!
                for (Map.Entry<Integer, EventDocWriteRequest> entry : vbucketToLastEvent.entrySet()) {
                    final int vbucket = entry.getKey();
                    Checkpoint checkpoint = entry.getValue().getEvent().getCheckpoint();
                    checkpoint = adjustForIgnoredEvents(vbucket, checkpoint);
                    checkpointService.set(entry.getKey(), checkpoint);
                }
                // were no writes for the same vbucket
                for (Map.Entry<Integer, Checkpoint> entry : ignoreBuffer.entrySet()) {
                    checkpointService.set(entry.getKey(), entry.getValue());
                }
                Metrics.bytesCounter().increment(totalEstimatedBytes);
                Metrics.indexTimePerDocument().record(indexingTookNanos / totalActionCount, NANOSECONDS);
                if (totalRetryDelayMillis != 0) {
                    Metrics.retryDelayTimer().record(totalRetryDelayMillis, MILLISECONDS);
                }
                if (LOGGER.isInfoEnabled()) {
                    final long elapsedMillis = NANOSECONDS.toMillis(System.nanoTime() - startNanos);
                    final ByteSizeValue prettySize = new ByteSizeValue(totalEstimatedBytes, ByteSizeUnit.BYTES);
                    LOGGER.info("Wrote {} actions ~{} in {} ms", totalActionCount, prettySize, elapsedMillis);
                }
                return;
            }
            // retry!
            retryReporter.report();
            Metrics.bulkRetriesCounter().increment();
            // todo check for hasNext? bail out or continue?
            final TimeValue retryDelay = waitIntervals.next();
            LOGGER.info("Retrying bulk request in {}", retryDelay);
            MILLISECONDS.sleep(retryDelay.millis());
            totalRetryDelayMillis += retryDelay.millis();
        }
    } finally {
        synchronized (this) {
            requestInProgress = false;
        }
    }
}

Example 2 with Checkpoint

use of com.couchbase.connector.dcp.Checkpoint in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointBackup method backup.

public static void backup(ConnectorConfig config, File outputFile) throws IOException {
    final Bucket bucket = CouchbaseHelper.openMetadataBucket(config.couchbase(), config.trustStore());
    final ResolvedBucketConfig bucketConfig = getBucketConfig(config.couchbase(), bucket);
    // don't care bucketConfig.uuid();
    final String bucketUuid = "";
    final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(getMetadataCollection(bucket, config.couchbase()), config.group().name());
    final int numVbuckets = bucketConfig.numberOfPartitions();
    final Set<Integer> vbuckets = IntStream.range(0, numVbuckets).boxed().collect(toSet());
    final Map<Integer, Checkpoint> checkpoints = checkpointDao.load(bucketUuid, vbuckets);
    final Map<String, Object> output = new LinkedHashMap<>();
    output.put("formatVersion", 1);
    output.put("bucketUuid", bucketConfig.uuid());
    output.put("vbuckets", checkpoints);
    atomicWrite(outputFile, tempFile -> {
        try (FileOutputStream out = new FileOutputStream(tempFile)) {
            new ObjectMapper().writeValue(out, output);
        }
    });
    System.out.println("Wrote checkpoint for connector '" + config.group().name() + "' to file " + outputFile.getAbsolutePath());
}

Example 3 with Checkpoint

use of com.couchbase.connector.dcp.Checkpoint in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointClear method setCheckpointToNow.

private static void setCheckpointToNow(ConnectorConfig config, Set<SeedNode> kvNodes, CheckpointDao checkpointDao) throws IOException {
    final Client dcpClient = DcpHelper.newClient(config.group().name(), config.couchbase(), kvNodes, config.trustStore());
    try {
        dcpClient.connect().block();
        final int numPartitions = dcpClient.numPartitions();
        final Set<Integer> allPartitions = new HashSet<>(allPartitions(numPartitions));
        DcpHelper.getCurrentSeqnos(dcpClient, allPartitions);
        final SessionState sessionState = dcpClient.sessionState();
        final Map<Integer, Checkpoint> now = new HashMap<>();
        for (int i = 0; i < allPartitions.size(); i++) {
            PartitionState p = sessionState.get(i);
            final long seqno = p.getStartSeqno();
            now.put(i, new Checkpoint(p.getLastUuid(), seqno, new SnapshotMarker(seqno, seqno)));
        }
        checkpointDao.save("", now);
    } finally {
        dcpClient.disconnect().block();
    }
}

Example 4 with Checkpoint

use of com.couchbase.connector.dcp.Checkpoint in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointRestore method restore.

public static void restore(ConnectorConfig config, File inputFile) throws IOException {
    final Bucket bucket = CouchbaseHelper.openMetadataBucket(config.couchbase(), config.trustStore());
    final ResolvedBucketConfig bucketConfig = getBucketConfig(config.couchbase(), bucket);
    final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(getMetadataCollection(bucket, config.couchbase()), config.group().name());
    final ObjectMapper mapper = new ObjectMapper();
    try (InputStream is = new FileInputStream(inputFile)) {
        final JsonNode json = mapper.readTree(is);
        final int formatVersion = json.path("formatVersion").intValue();
        final String bucketUuid = json.path("bucketUuid").asText();
        if (formatVersion != 1) {
            throw new IllegalArgumentException("Unrecognized checkpoint format version: " + formatVersion);
        }
        if (!bucketUuid.equals(bucketConfig.uuid())) {
            throw new IllegalArgumentException("Bucket UUID mismatch; checkpoint is from a bucket with UUID " + bucketUuid + " but this bucket has UUID " + bucketConfig.uuid());
        }
        final Map<Integer, Checkpoint> checkpoints = mapper.convertValue(json.get("vbuckets"), new TypeReference<Map<Integer, Checkpoint>>() {
        });
        if (checkpoints.size() != bucketConfig.numberOfPartitions()) {
            throw new IllegalArgumentException("Bucket has " + bucketConfig.numberOfPartitions() + " vbuckets but the checkpoint file has " + checkpoints.size() + " -- is it from a different operating system (for example, macOS vs Linux)?");
        }
        checkpointDao.save("", checkpoints);
    }
    System.out.println("Restored checkpoint for connector '" + config.group().name() + "' from file " + inputFile);
}

Example 5 with Checkpoint

use of com.couchbase.connector.dcp.Checkpoint in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointClear method run.

private static void run(ConnectorConfig config, boolean catchUp) throws IOException {
    final ClusterEnvironment env = environmentBuilder(config.couchbase(), config.trustStore()).build();
    final Cluster cluster = createCluster(config.couchbase(), env);
    try {
        final Bucket metadataBucket = CouchbaseHelper.waitForBucket(cluster, config.couchbase().metadataBucket());
        final Collection metadataCollection = CouchbaseHelper.getMetadataCollection(metadataBucket, config.couchbase());
        final ResolvedBucketConfig bucketConfig = getBucketConfig(config.couchbase(), metadataBucket);
        final Set<SeedNode> kvNodes = getKvNodes(config.couchbase(), metadataBucket);
        final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(metadataCollection, config.group().name());
        if (catchUp) {
            setCheckpointToNow(config, kvNodes, checkpointDao);
            System.out.println("Set checkpoint for connector '" + config.group().name() + "' to match current state of Couchbase bucket.");
        } else {
            final int numVbuckets = bucketConfig.numberOfPartitions();
            final Set<Integer> vbuckets = IntStream.range(0, numVbuckets).boxed().collect(toSet());
            checkpointDao.clear(bucketConfig.uuid(), vbuckets);
            System.out.println("Cleared checkpoint for connector '" + config.group().name() + "'.");
        }
    } finally {
        cluster.disconnect();
        env.shutdown();
    }
}