Search in sources :

Example 1 with PanicButton

use of com.couchbase.connector.cluster.PanicButton in project couchbase-elasticsearch-connector by couchbase.

the class ElasticsearchConnector method main.

public static void main(String... args) throws Throwable {
    LOGGER.info("Couchbase Elasticsearch Connector version {}", getVersionString());
    final OptionsParser parser = new OptionsParser();
    final OptionSet options = parser.parse(args);
    final File configFile = options.valueOf(parser.configFile);
    System.out.println("Reading connector configuration from " + configFile.getAbsoluteFile());
    ConnectorConfig config = ConnectorConfig.from(configFile);
    final PanicButton panicButton = new DefaultPanicButton();
    boolean watchK8sReplicas = "true".equals(System.getenv("CBES_K8S_WATCH_REPLICAS"));
    boolean getMemberNumberFromHostname = watchK8sReplicas || "true".equals(System.getenv("CBES_K8S_STATEFUL_SET"));
    if (getMemberNumberFromHostname) {
        int memberNumber = StatefulSetInfo.fromHostname().podOrdinal + 1;
        LOGGER.info("Getting group member number from Kubernetes pod hostname: {}", memberNumber);
        // This is a kludge. The Membership class validates its arguments, so you can't have a Membership
        // of "4 of 1", for example. If we plan to get the group size from the Kubernetes StatefulSet,
        // bypass this validation by temporarily setting the group size to the largest sane value (1024).
        // We'll dial it down to the actual size of the StatefulSet a bit later on.
        int clusterSize = watchK8sReplicas ? 1024 : config.group().staticMembership().getClusterSize();
        config = transformMembership(config, m -> Membership.of(memberNumber, clusterSize));
    }
    KubernetesClient k8sClient = null;
    try {
        if (watchK8sReplicas) {
            k8sClient = new DefaultKubernetesClient();
            LOGGER.info("Activating native Kubernetes integration; connector will use StatefulSet spec" + " to determine group size." + " This mode requires a Kubernetes service account with 'get' and 'watch', and 'list'" + " permissions for the StatefulSet.");
            int k8sReplicas = ReplicaChangeWatcher.getReplicasAndPanicOnChange(k8sClient, panicButton);
            config = transformMembership(config, m -> Membership.of(m.getMemberNumber(), k8sReplicas));
        }
        if (watchK8sReplicas || getMemberNumberFromHostname) {
            LOGGER.info("Patched configuration with info from Kubernetes environment; membership = {}", config.group().staticMembership());
        }
        if (config.group().staticMembership().getClusterSize() > 1024) {
            panicButton.panic("Invalid group size configuration; totalMembers must be <= 1024." + " Did you forget to set the CBES_TOTAL_MEMBERS environment variable?");
        }
        Duration startupQuietPeriod = watchK8sReplicas ? ReplicaChangeWatcher.startupQuietPeriod() : Duration.ZERO;
        run(config, panicButton, startupQuietPeriod);
    } finally {
        if (k8sClient != null) {
            // so client threads don't prevent app from exiting
            k8sClient.close();
        }
    }
}
Also used : DcpHelper.initSessionState(com.couchbase.connector.dcp.DcpHelper.initSessionState) LoggerFactory(org.slf4j.LoggerFactory) ImmutableConnectorConfig(com.couchbase.connector.config.es.ImmutableConnectorConfig) Collection(com.couchbase.client.java.Collection) Duration(java.time.Duration) ClusterEnvironment(com.couchbase.client.java.env.ClusterEnvironment) RequestFactory(com.couchbase.connector.elasticsearch.io.RequestFactory) DefaultKubernetesClient(io.fabric8.kubernetes.client.DefaultKubernetesClient) DcpHelper.initEventListener(com.couchbase.connector.dcp.DcpHelper.initEventListener) ConnectorConfig(com.couchbase.connector.config.es.ConnectorConfig) OptionSet(joptsimple.OptionSet) StreamFrom(com.couchbase.client.dcp.StreamFrom) StatefulSetInfo(com.couchbase.connector.cluster.k8s.StatefulSetInfo) StreamTo(com.couchbase.client.dcp.StreamTo) LogLevel(com.couchbase.client.dcp.metrics.LogLevel) Set(java.util.Set) Version(com.couchbase.client.dcp.util.Version) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Membership(com.couchbase.connector.cluster.Membership) Executors(java.util.concurrent.Executors) RuntimeHelper(com.couchbase.connector.util.RuntimeHelper) Bucket(com.couchbase.client.java.Bucket) ElasticsearchConfig(com.couchbase.connector.config.es.ElasticsearchConfig) ThrowableHelper(com.couchbase.connector.util.ThrowableHelper) Slf4jReporter(com.codahale.metrics.Slf4jReporter) CheckpointService(com.couchbase.connector.dcp.CheckpointService) ConfigException(com.couchbase.connector.config.ConfigException) TypeConfig(com.couchbase.connector.config.es.TypeConfig) ImmutableGroupConfig(com.couchbase.connector.config.common.ImmutableGroupConfig) ElasticsearchHelper.newElasticsearchClient(com.couchbase.connector.elasticsearch.ElasticsearchHelper.newElasticsearchClient) HttpServer(com.couchbase.connector.util.HttpServer) Client(com.couchbase.client.dcp.Client) LogRedaction(com.couchbase.client.core.logging.LogRedaction) SeedNode(com.couchbase.client.core.env.SeedNode) CheckpointDao(com.couchbase.connector.dcp.CheckpointDao) Function(java.util.function.Function) DefaultPanicButton(com.couchbase.connector.cluster.DefaultPanicButton) TimeValue(org.elasticsearch.common.unit.TimeValue) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) AbstractCliCommand(com.couchbase.connector.elasticsearch.cli.AbstractCliCommand) Logger(org.slf4j.Logger) RedactableArgument.redactSystem(com.couchbase.client.core.logging.RedactableArgument.redactSystem) PanicButton(com.couchbase.connector.cluster.PanicButton) CouchbaseCheckpointDao(com.couchbase.connector.dcp.CouchbaseCheckpointDao) DcpHelper(com.couchbase.connector.dcp.DcpHelper) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) File(java.io.File) Cluster(com.couchbase.client.java.Cluster) CouchbaseHelper(com.couchbase.connector.dcp.CouchbaseHelper) ReplicaChangeWatcher(com.couchbase.connector.cluster.k8s.ReplicaChangeWatcher) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) ElasticsearchHelper.waitForElasticsearchAndRequireVersion(com.couchbase.connector.elasticsearch.ElasticsearchHelper.waitForElasticsearchAndRequireVersion) SECONDS(java.util.concurrent.TimeUnit.SECONDS) VersionHelper.getVersionString(com.couchbase.connector.VersionHelper.getVersionString) DefaultKubernetesClient(io.fabric8.kubernetes.client.DefaultKubernetesClient) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) ImmutableConnectorConfig(com.couchbase.connector.config.es.ImmutableConnectorConfig) ConnectorConfig(com.couchbase.connector.config.es.ConnectorConfig) DefaultPanicButton(com.couchbase.connector.cluster.DefaultPanicButton) PanicButton(com.couchbase.connector.cluster.PanicButton) Duration(java.time.Duration) DefaultKubernetesClient(io.fabric8.kubernetes.client.DefaultKubernetesClient) OptionSet(joptsimple.OptionSet) File(java.io.File) DefaultPanicButton(com.couchbase.connector.cluster.DefaultPanicButton)

Aggregations

Slf4jReporter (com.codahale.metrics.Slf4jReporter)1 SeedNode (com.couchbase.client.core.env.SeedNode)1 LogRedaction (com.couchbase.client.core.logging.LogRedaction)1 RedactableArgument.redactSystem (com.couchbase.client.core.logging.RedactableArgument.redactSystem)1 Client (com.couchbase.client.dcp.Client)1 StreamFrom (com.couchbase.client.dcp.StreamFrom)1 StreamTo (com.couchbase.client.dcp.StreamTo)1 LogLevel (com.couchbase.client.dcp.metrics.LogLevel)1 Version (com.couchbase.client.dcp.util.Version)1 Bucket (com.couchbase.client.java.Bucket)1 Cluster (com.couchbase.client.java.Cluster)1 Collection (com.couchbase.client.java.Collection)1 ClusterEnvironment (com.couchbase.client.java.env.ClusterEnvironment)1 VersionHelper.getVersionString (com.couchbase.connector.VersionHelper.getVersionString)1 DefaultPanicButton (com.couchbase.connector.cluster.DefaultPanicButton)1 Membership (com.couchbase.connector.cluster.Membership)1 PanicButton (com.couchbase.connector.cluster.PanicButton)1 ReplicaChangeWatcher (com.couchbase.connector.cluster.k8s.ReplicaChangeWatcher)1 StatefulSetInfo (com.couchbase.connector.cluster.k8s.StatefulSetInfo)1 ConfigException (com.couchbase.connector.config.ConfigException)1