Examples with SeedNode com.couchbase.client.core.env.SeedNode used on opensource projects

Example 1 with SeedNode

use of com.couchbase.client.core.env.SeedNode in project ShedLock by lukas-krecan.

the class CouchbaseLockProviderIntegrationTest method startCouchbase.

@BeforeAll
public static void startCouchbase() {
    container = new CouchbaseContainer().withBucket(new BucketDefinition(BUCKET_NAME));
    container.start();
    Set<SeedNode> seedNodes = new HashSet<>(Arrays.asList(SeedNode.create(container.getContainerIpAddress(), Optional.of(container.getBootstrapCarrierDirectPort()), Optional.of(container.getBootstrapHttpDirectPort()))));
    ClusterOptions options = ClusterOptions.clusterOptions(container.getUsername(), container.getPassword());
    cluster = Cluster.connect(seedNodes, options);
    bucket = cluster.bucket(BUCKET_NAME);
    bucket.waitUntilReady(Duration.ofSeconds(30));
}

Example 2 with SeedNode

use of com.couchbase.client.core.env.SeedNode in project couchbase-elasticsearch-connector by couchbase.

the class SeedNodeHelper method getKvNodes.

/**
 * Returns complete host and port information for every node running the KV service.
 */
public static Set<SeedNode> getKvNodes(Bucket bucket, String connectionString, boolean tls, NetworkResolution networkResolution, Duration timeout) {
    // Bucket config is most reliable way to determine KV ports
    BucketConfig config = CouchbaseHelper.getConfig(bucket, timeout);
    // so the actual node names can be used.
    if (networkResolution.equals(NetworkResolution.AUTO)) {
        EventBus eventBus = bucket.environment().eventBus();
        // The Java client calls seedNodesFromConnectionString during bootstrap, but the results
        // are not accessible here. Call it again.
        Set<String> seedHosts = ConnectionStringUtil.seedNodesFromConnectionString(connectionString, true, tls, eventBus).stream().map(SeedNode::address).collect(toSet());
        networkResolution = inferNetwork(config, networkResolution, seedHosts);
    }
    // Resolve alternate addresses
    // so lambda can access it
    NetworkResolution finalNetworkResolution = networkResolution;
    return config.nodes().stream().map(node -> new ResolvedNodeInfo(node, tls, finalNetworkResolution)).map(node -> SeedNode.create(node.host(), node.port(KV), node.port(MANAGER))).filter(node -> node.kvPort().isPresent()).collect(toSet());
}

Example 3 with SeedNode

use of com.couchbase.client.core.env.SeedNode in project couchbase-elasticsearch-connector by couchbase.

the class ElasticsearchConnector method run.

public static void run(ConnectorConfig config, PanicButton panicButton, Duration startupQuietPeriod) throws Throwable {
    final Throwable fatalError;
    final Membership membership = config.group().staticMembership();
    LOGGER.info("Read configuration: {}", redactSystem(config));
    final ScheduledExecutorService checkpointExecutor = Executors.newSingleThreadScheduledExecutor();
    try (Slf4jReporter metricReporter = newSlf4jReporter(config.metrics().logInterval());
        HttpServer httpServer = new HttpServer(config.metrics().httpPort(), membership);
        RestHighLevelClient esClient = newElasticsearchClient(config.elasticsearch(), config.trustStore())) {
        DocumentLifecycle.setLogLevel(config.logging().logDocumentLifecycle() ? LogLevel.INFO : LogLevel.DEBUG);
        LogRedaction.setRedactionLevel(config.logging().redactionLevel());
        DcpHelper.setRedactionLevel(config.logging().redactionLevel());
        final ClusterEnvironment env = CouchbaseHelper.environmentBuilder(config.couchbase(), config.trustStore()).build();
        final Cluster cluster = CouchbaseHelper.createCluster(config.couchbase(), env);
        final Version elasticsearchVersion = waitForElasticsearchAndRequireVersion(esClient, new Version(2, 0, 0), new Version(5, 6, 16));
        LOGGER.info("Elasticsearch version {}", elasticsearchVersion);
        validateConfig(elasticsearchVersion, config.elasticsearch());
        // Wait for couchbase server to come online, then open the bucket.
        final Bucket bucket = CouchbaseHelper.waitForBucket(cluster, config.couchbase().bucket());
        final Set<SeedNode> kvNodes = CouchbaseHelper.getKvNodes(config.couchbase(), bucket);
        final boolean storeMetadataInSourceBucket = config.couchbase().metadataBucket().equals(config.couchbase().bucket());
        final Bucket metadataBucket = storeMetadataInSourceBucket ? bucket : CouchbaseHelper.waitForBucket(cluster, config.couchbase().metadataBucket());
        final Collection metadataCollection = CouchbaseHelper.getMetadataCollection(metadataBucket, config.couchbase());
        final CheckpointDao checkpointDao = new CouchbaseCheckpointDao(metadataCollection, config.group().name());
        // todo get this from dcp client
        final String bucketUuid = "";
        final CheckpointService checkpointService = new CheckpointService(bucketUuid, checkpointDao);
        final RequestFactory requestFactory = new RequestFactory(config.elasticsearch().types(), config.elasticsearch().docStructure(), config.elasticsearch().rejectLog());
        final ElasticsearchWorkerGroup workers = new ElasticsearchWorkerGroup(esClient, checkpointService, requestFactory, ErrorListener.NOOP, config.elasticsearch().bulkRequest());
        Metrics.gauge("write.queue", "Document events currently buffered in memory.", workers, ElasticsearchWorkerGroup::getQueueSize);
        // High value indicates the connector has stalled
        Metrics.gauge("es.wait.ms", null, workers, ElasticsearchWorkerGroup::getCurrentRequestMillis);
        // Same as "es.wait.ms" but normalized to seconds for Prometheus
        Metrics.gauge("es.wait.seconds", "Duration of in-flight Elasticsearch bulk request (including any retries). Long duration may indicate connector has stalled.", workers, value -> value.getCurrentRequestMillis() / (double) SECONDS.toMillis(1));
        final Client dcpClient = DcpHelper.newClient(config.group().name(), config.couchbase(), kvNodes, config.trustStore());
        initEventListener(dcpClient, panicButton, workers::submit);
        final Thread saveCheckpoints = new Thread(checkpointService::save, "save-checkpoints");
        try {
            try {
                dcpClient.connect().block(Duration.ofMillis(config.couchbase().dcp().connectTimeout().millis()));
            } catch (Throwable t) {
                panicButton.panic("Failed to establish initial DCP connection within " + config.couchbase().dcp().connectTimeout(), t);
            }
            final int numPartitions = dcpClient.numPartitions();
            LOGGER.info("Bucket has {} partitions. Membership = {}", numPartitions, membership);
            final Set<Integer> partitions = membership.getPartitions(numPartitions);
            if (partitions.isEmpty()) {
                // need to do this check, because if we started streaming with an empty list, the DCP client would open streams for *all* partitions
                throw new IllegalArgumentException("There are more workers than Couchbase vbuckets; this worker doesn't have any work to do.");
            }
            checkpointService.init(numPartitions, () -> DcpHelper.getCurrentSeqnosAsMap(dcpClient, partitions, Duration.ofSeconds(5)));
            dcpClient.initializeState(StreamFrom.BEGINNING, StreamTo.INFINITY).block();
            initSessionState(dcpClient, checkpointService, partitions);
            // configuration problems.
            if (!startupQuietPeriod.isZero()) {
                LOGGER.info("Entering startup quiet period; sleeping for {} so peers can terminate in case of unsafe scaling.", startupQuietPeriod);
                MILLISECONDS.sleep(startupQuietPeriod.toMillis());
                LOGGER.info("Startup quiet period complete.");
            }
            checkpointExecutor.scheduleWithFixedDelay(checkpointService::save, 10, 10, SECONDS);
            RuntimeHelper.addShutdownHook(saveCheckpoints);
            // Unless shutdown is due to panic...
            panicButton.addPrePanicHook(() -> RuntimeHelper.removeShutdownHook(saveCheckpoints));
            try {
                LOGGER.debug("Opening DCP streams for partitions: {}", partitions);
                dcpClient.startStreaming(partitions).block();
            } catch (RuntimeException e) {
                ThrowableHelper.propagateCauseIfPossible(e, InterruptedException.class);
                throw e;
            }
            // Start HTTP server *after* other setup is complete, so the metrics endpoint
            // can be used as a "successful startup" probe.
            httpServer.start();
            if (config.metrics().httpPort() >= 0) {
                LOGGER.info("Prometheus metrics available at http://localhost:{}/metrics/prometheus", httpServer.getBoundPort());
                LOGGER.info("Dropwizard metrics available at http://localhost:{}/metrics/dropwizard?pretty", httpServer.getBoundPort());
            } else {
                LOGGER.info("Metrics HTTP server is disabled. Edit the [metrics] 'httpPort' config property to enable.");
            }
            LOGGER.info("Elasticsearch connector startup complete.");
            fatalError = workers.awaitFatalError();
            LOGGER.error("Terminating due to fatal error from worker", fatalError);
        } catch (InterruptedException shutdownRequest) {
            LOGGER.info("Graceful shutdown requested. Saving checkpoints and cleaning up.");
            checkpointService.save();
            throw shutdownRequest;
        } catch (Throwable t) {
            LOGGER.error("Terminating due to fatal error during setup", t);
            throw t;
        } finally {
            // If we get here it means there was a fatal exception, or the connector is running in distributed
            // or test mode and a graceful shutdown was requested. Don't need the shutdown hook for any of those cases.
            RuntimeHelper.removeShutdownHook(saveCheckpoints);
            checkpointExecutor.shutdown();
            metricReporter.stop();
            dcpClient.disconnect().block();
            // to avoid buffer leak, must close *after* dcp client stops feeding it events
            workers.close();
            checkpointExecutor.awaitTermination(10, SECONDS);
            cluster.disconnect();
            // can't reuse, because connector config might have different SSL settings next time
            env.shutdown();
        }
    }
    // give stdout a chance to quiet down so the stack trace on stderr isn't interleaved with stdout.
    MILLISECONDS.sleep(500);
    throw fatalError;
}

Example 4 with SeedNode

use of com.couchbase.client.core.env.SeedNode in project kafka-connect-couchbase by couchbase.

the class SeedNodeHelper method getKvNodes.

/**
 * Returns complete host and port information for every node running the KV service.
 */
public static Set<SeedNode> getKvNodes(Bucket bucket, String connectionString, boolean tls, NetworkResolution networkResolution, Duration timeout) {
    // Bucket config is most reliable way to determine KV ports
    BucketConfig config = CouchbaseHelper.getConfig(bucket, timeout);
    // so the actual node names can be used.
    if (networkResolution.equals(NetworkResolution.AUTO)) {
        EventBus eventBus = bucket.environment().eventBus();
        // The Java client calls seedNodesFromConnectionString during bootstrap, but the results
        // are not accessible here. Call it again.
        Set<String> seedHosts = ConnectionStringUtil.seedNodesFromConnectionString(connectionString, true, tls, eventBus).stream().map(SeedNode::address).collect(toSet());
        networkResolution = inferNetwork(config, networkResolution, seedHosts);
    }
    // Resolve alternate addresses
    // so lambda can access it
    NetworkResolution finalNetworkResolution = networkResolution;
    return config.nodes().stream().map(node -> new ResolvedNodeInfo(node, tls, finalNetworkResolution)).map(node -> SeedNode.create(node.host(), node.port(KV), node.port(MANAGER))).filter(node -> node.kvPort().isPresent()).collect(toSet());
}

Example 5 with SeedNode

use of com.couchbase.client.core.env.SeedNode in project couchbase-jvm-clients by couchbase.

the class DefaultConfigurationProviderIntegrationTest method openBucketFromSecondValidSeed.

/**
 * Bucket config should also be loaded when the first seed in the list is not available.
 */
@Test
void openBucketFromSecondValidSeed() {
    TestNodeConfig cfg = config().firstNodeWith(Services.KV).get();
    Set<SeedNode> seeds = new HashSet<>(Arrays.asList(SeedNode.create("1.2.3.4"), SeedNode.create(cfg.hostname(), Optional.of(cfg.ports().get(Services.KV)), Optional.of(cfg.ports().get(Services.MANAGER)))));
    SimpleEventBus eventBus = new SimpleEventBus(true);
    environment = CoreEnvironment.builder().eventBus(eventBus).build();
    core = Core.create(environment, authenticator(), seeds);
    String bucketName = config().bucketname();
    ConfigurationProvider provider = new DefaultConfigurationProvider(core, seeds);
    openAndClose(bucketName, provider);
    provider.shutdown().block();
    waitUntilCondition(() -> eventBus.publishedEvents().stream().anyMatch(e -> e instanceof EndpointConnectionFailedEvent));
}