Examples with CheckpointDao com.couchbase.connector.dcp.CheckpointDao used on opensource projects

Example 1 with CheckpointDao

use of com.couchbase.connector.dcp.CheckpointDao in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointBackup method backup.

public static void backup(ConnectorConfig config, File outputFile) throws IOException {
    final Bucket bucket = CouchbaseHelper.openMetadataBucket(config.couchbase(), config.trustStore());
    final ResolvedBucketConfig bucketConfig = getBucketConfig(config.couchbase(), bucket);
    // don't care bucketConfig.uuid();
    final String bucketUuid = "";
    final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(getMetadataCollection(bucket, config.couchbase()), config.group().name());
    final int numVbuckets = bucketConfig.numberOfPartitions();
    final Set<Integer> vbuckets = IntStream.range(0, numVbuckets).boxed().collect(toSet());
    final Map<Integer, Checkpoint> checkpoints = checkpointDao.load(bucketUuid, vbuckets);
    final Map<String, Object> output = new LinkedHashMap<>();
    output.put("formatVersion", 1);
    output.put("bucketUuid", bucketConfig.uuid());
    output.put("vbuckets", checkpoints);
    atomicWrite(outputFile, tempFile -> {
        try (FileOutputStream out = new FileOutputStream(tempFile)) {
            new ObjectMapper().writeValue(out, output);
        }
    });
    System.out.println("Wrote checkpoint for connector '" + config.group().name() + "' to file " + outputFile.getAbsolutePath());
}

Example 2 with CheckpointDao

use of com.couchbase.connector.dcp.CheckpointDao in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointRestore method restore.

public static void restore(ConnectorConfig config, File inputFile) throws IOException {
    final Bucket bucket = CouchbaseHelper.openMetadataBucket(config.couchbase(), config.trustStore());
    final ResolvedBucketConfig bucketConfig = getBucketConfig(config.couchbase(), bucket);
    final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(getMetadataCollection(bucket, config.couchbase()), config.group().name());
    final ObjectMapper mapper = new ObjectMapper();
    try (InputStream is = new FileInputStream(inputFile)) {
        final JsonNode json = mapper.readTree(is);
        final int formatVersion = json.path("formatVersion").intValue();
        final String bucketUuid = json.path("bucketUuid").asText();
        if (formatVersion != 1) {
            throw new IllegalArgumentException("Unrecognized checkpoint format version: " + formatVersion);
        }
        if (!bucketUuid.equals(bucketConfig.uuid())) {
            throw new IllegalArgumentException("Bucket UUID mismatch; checkpoint is from a bucket with UUID " + bucketUuid + " but this bucket has UUID " + bucketConfig.uuid());
        }
        final Map<Integer, Checkpoint> checkpoints = mapper.convertValue(json.get("vbuckets"), new TypeReference<Map<Integer, Checkpoint>>() {
        });
        if (checkpoints.size() != bucketConfig.numberOfPartitions()) {
            throw new IllegalArgumentException("Bucket has " + bucketConfig.numberOfPartitions() + " vbuckets but the checkpoint file has " + checkpoints.size() + " -- is it from a different operating system (for example, macOS vs Linux)?");
        }
        checkpointDao.save("", checkpoints);
    }
    System.out.println("Restored checkpoint for connector '" + config.group().name() + "' from file " + inputFile);
}

Example 3 with CheckpointDao

use of com.couchbase.connector.dcp.CheckpointDao in project couchbase-elasticsearch-connector by couchbase.

the class ElasticsearchConnector method run.

public static void run(ConnectorConfig config, PanicButton panicButton, Duration startupQuietPeriod) throws Throwable {
    final Throwable fatalError;
    final Membership membership = config.group().staticMembership();
    LOGGER.info("Read configuration: {}", redactSystem(config));
    final ScheduledExecutorService checkpointExecutor = Executors.newSingleThreadScheduledExecutor();
    try (Slf4jReporter metricReporter = newSlf4jReporter(config.metrics().logInterval());
        HttpServer httpServer = new HttpServer(config.metrics().httpPort(), membership);
        RestHighLevelClient esClient = newElasticsearchClient(config.elasticsearch(), config.trustStore())) {
        DocumentLifecycle.setLogLevel(config.logging().logDocumentLifecycle() ? LogLevel.INFO : LogLevel.DEBUG);
        LogRedaction.setRedactionLevel(config.logging().redactionLevel());
        DcpHelper.setRedactionLevel(config.logging().redactionLevel());
        final ClusterEnvironment env = CouchbaseHelper.environmentBuilder(config.couchbase(), config.trustStore()).build();
        final Cluster cluster = CouchbaseHelper.createCluster(config.couchbase(), env);
        final Version elasticsearchVersion = waitForElasticsearchAndRequireVersion(esClient, new Version(2, 0, 0), new Version(5, 6, 16));
        LOGGER.info("Elasticsearch version {}", elasticsearchVersion);
        validateConfig(elasticsearchVersion, config.elasticsearch());
        // Wait for couchbase server to come online, then open the bucket.
        final Bucket bucket = CouchbaseHelper.waitForBucket(cluster, config.couchbase().bucket());
        final Set<SeedNode> kvNodes = CouchbaseHelper.getKvNodes(config.couchbase(), bucket);
        final boolean storeMetadataInSourceBucket = config.couchbase().metadataBucket().equals(config.couchbase().bucket());
        final Bucket metadataBucket = storeMetadataInSourceBucket ? bucket : CouchbaseHelper.waitForBucket(cluster, config.couchbase().metadataBucket());
        final Collection metadataCollection = CouchbaseHelper.getMetadataCollection(metadataBucket, config.couchbase());
        final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(metadataCollection, config.group().name());
        // todo get this from dcp client
        final String bucketUuid = "";
        final CheckpointService checkpointService = new CheckpointService(bucketUuid, checkpointDao);
        final RequestFactory requestFactory = new RequestFactory(config.elasticsearch().types(), config.elasticsearch().docStructure(), config.elasticsearch().rejectLog());
        final ElasticsearchWorkerGroup workers = new ElasticsearchWorkerGroup(esClient, checkpointService, requestFactory, ErrorListener.NOOP, config.elasticsearch().bulkRequest());
        Metrics.gauge("write.queue", "Document events currently buffered in memory.", workers, ElasticsearchWorkerGroup::getQueueSize);
        // High value indicates the connector has stalled
        Metrics.gauge("es.wait.ms", null, workers, ElasticsearchWorkerGroup::getCurrentRequestMillis);
        // Same as "es.wait.ms" but normalized to seconds for Prometheus
        Metrics.gauge("es.wait.seconds", "Duration of in-flight Elasticsearch bulk request (including any retries). Long duration may indicate connector has stalled.", workers, value -> value.getCurrentRequestMillis() / (double) SECONDS.toMillis(1));
        final Client dcpClient = DcpHelper.newClient(config.group().name(), config.couchbase(), kvNodes, config.trustStore());
        initEventListener(dcpClient, panicButton, workers::submit);
        final Thread saveCheckpoints = new Thread(checkpointService::save, "save-checkpoints");
        try {
            try {
                dcpClient.connect().block(Duration.ofMillis(config.couchbase().dcp().connectTimeout().millis()));
            } catch (Throwable t) {
                panicButton.panic("Failed to establish initial DCP connection within " + config.couchbase().dcp().connectTimeout(), t);
            }
            final int numPartitions = dcpClient.numPartitions();
            LOGGER.info("Bucket has {} partitions. Membership = {}", numPartitions, membership);
            final Set<Integer> partitions = membership.getPartitions(numPartitions);
            if (partitions.isEmpty()) {
                // need to do this check, because if we started streaming with an empty list, the DCP client would open streams for *all* partitions
                throw new IllegalArgumentException("There are more workers than Couchbase vbuckets; this worker doesn't have any work to do.");
            }
            checkpointService.init(numPartitions, () -> DcpHelper.getCurrentSeqnosAsMap(dcpClient, partitions, Duration.ofSeconds(5)));
            dcpClient.initializeState(StreamFrom.BEGINNING, StreamTo.INFINITY).block();
            initSessionState(dcpClient, checkpointService, partitions);
            // configuration problems.
            if (!startupQuietPeriod.isZero()) {
                LOGGER.info("Entering startup quiet period; sleeping for {} so peers can terminate in case of unsafe scaling.", startupQuietPeriod);
                MILLISECONDS.sleep(startupQuietPeriod.toMillis());
                LOGGER.info("Startup quiet period complete.");
            }
            checkpointExecutor.scheduleWithFixedDelay(checkpointService::save, 10, 10, SECONDS);
            RuntimeHelper.addShutdownHook(saveCheckpoints);
            // Unless shutdown is due to panic...
            panicButton.addPrePanicHook(() -> RuntimeHelper.removeShutdownHook(saveCheckpoints));
            try {
                LOGGER.debug("Opening DCP streams for partitions: {}", partitions);
                dcpClient.startStreaming(partitions).block();
            } catch (RuntimeException e) {
                ThrowableHelper.propagateCauseIfPossible(e, InterruptedException.class);
                throw e;
            }
            // Start HTTP server *after* other setup is complete, so the metrics endpoint
            // can be used as a "successful startup" probe.
            httpServer.start();
            if (config.metrics().httpPort() >= 0) {
                LOGGER.info("Prometheus metrics available at http://localhost:{}/metrics/prometheus", httpServer.getBoundPort());
                LOGGER.info("Dropwizard metrics available at http://localhost:{}/metrics/dropwizard?pretty", httpServer.getBoundPort());
            } else {
                LOGGER.info("Metrics HTTP server is disabled. Edit the [metrics] 'httpPort' config property to enable.");
            }
            LOGGER.info("Elasticsearch connector startup complete.");
            fatalError = workers.awaitFatalError();
            LOGGER.error("Terminating due to fatal error from worker", fatalError);
        } catch (InterruptedException shutdownRequest) {
            LOGGER.info("Graceful shutdown requested. Saving checkpoints and cleaning up.");
            checkpointService.save();
            throw shutdownRequest;
        } catch (Throwable t) {
            LOGGER.error("Terminating due to fatal error during setup", t);
            throw t;
        } finally {
            // If we get here it means there was a fatal exception, or the connector is running in distributed
            // or test mode and a graceful shutdown was requested. Don't need the shutdown hook for any of those cases.
            RuntimeHelper.removeShutdownHook(saveCheckpoints);
            checkpointExecutor.shutdown();
            metricReporter.stop();
            dcpClient.disconnect().block();
            // to avoid buffer leak, must close *after* dcp client stops feeding it events
            workers.close();
            checkpointExecutor.awaitTermination(10, SECONDS);
            cluster.disconnect();
            // can't reuse, because connector config might have different SSL settings next time
            env.shutdown();
        }
    }
    // give stdout a chance to quiet down so the stack trace on stderr isn't interleaved with stdout.
    MILLISECONDS.sleep(500);
    throw fatalError;
}

Example 4 with CheckpointDao

use of com.couchbase.connector.dcp.CheckpointDao in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointClear method run.

private static void run(ConnectorConfig config, boolean catchUp) throws IOException {
    final ClusterEnvironment env = environmentBuilder(config.couchbase(), config.trustStore()).build();
    final Cluster cluster = createCluster(config.couchbase(), env);
    try {
        final Bucket metadataBucket = CouchbaseHelper.waitForBucket(cluster, config.couchbase().metadataBucket());
        final Collection metadataCollection = CouchbaseHelper.getMetadataCollection(metadataBucket, config.couchbase());
        final ResolvedBucketConfig bucketConfig = getBucketConfig(config.couchbase(), metadataBucket);
        final Set<SeedNode> kvNodes = getKvNodes(config.couchbase(), metadataBucket);
        final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(metadataCollection, config.group().name());
        if (catchUp) {
            setCheckpointToNow(config, kvNodes, checkpointDao);
            System.out.println("Set checkpoint for connector '" + config.group().name() + "' to match current state of Couchbase bucket.");
        } else {
            final int numVbuckets = bucketConfig.numberOfPartitions();
            final Set<Integer> vbuckets = IntStream.range(0, numVbuckets).boxed().collect(toSet());
            checkpointDao.clear(bucketConfig.uuid(), vbuckets);
            System.out.println("Cleared checkpoint for connector '" + config.group().name() + "'.");
        }
    } finally {
        cluster.disconnect();
        env.shutdown();
    }
}