Examples with ConnectorConfig com.couchbase.connector.config.es.ConnectorConfig used on opensource projects

Example 1 with ConnectorConfig

use of com.couchbase.connector.config.es.ConnectorConfig in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointClear method main.

public static void main(String[] args) throws Exception {
    final OptionsParser parser = new OptionsParser();
    final OptionSet options = parser.parse(args);
    final File configFile = options.valueOf(parser.configFile);
    System.out.println("Reading connector configuration from " + configFile.getAbsoluteFile());
    final ConnectorConfig config = ConnectorConfig.from(configFile);
    run(config, options.has(parser.catchUp));
}

Example 2 with ConnectorConfig

use of com.couchbase.connector.config.es.ConnectorConfig in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointRestore method main.

public static void main(String[] args) throws Exception {
    final OptionParser parser = new OptionParser();
    final OptionSet options = parser.parse(args);
    final File configFile = options.valueOf(parser.configFile);
    System.out.println("Reading connector configuration from " + configFile.getAbsoluteFile());
    final ConnectorConfig config = ConnectorConfig.from(configFile);
    final File inputFile = options.valueOf(parser.inputFile);
    restore(config, inputFile);
}

Example 3 with ConnectorConfig

use of com.couchbase.connector.config.es.ConnectorConfig in project couchbase-elasticsearch-connector by couchbase.

the class WorkerServiceImpl method startStreaming.

@Override
public synchronized void startStreaming(Membership membership, String config) {
    stopStreaming();
    startKillSwitchTimer();
    final ImmutableConnectorConfig originalConfig = ConnectorConfig.from(config);
    // Plug in the appropriate group membership. Ick.
    final ConnectorConfig patchedConfig = originalConfig.withGroup(ImmutableGroupConfig.copyOf(originalConfig.group()).withStaticMembership(membership));
    connectorTask = AsyncTask.run(() -> ElasticsearchConnector.run(patchedConfig), fatalErrorListener);
    this.status = new Status(membership);
}

Example 4 with ConnectorConfig

use of com.couchbase.connector.config.es.ConnectorConfig in project couchbase-elasticsearch-connector by couchbase.

the class CheckpointBackup method main.

public static void main(String[] args) throws Exception {
    final OptionsParser parser = new OptionsParser();
    final OptionSet options = parser.parse(args);
    final File configFile = options.valueOf(parser.configFile);
    final File outputFile = options.valueOf(parser.outputFile);
    // if (outputFile.exists() && !options.has(parser.forceOverwrite)) {
    // System.err.println("ERROR: Must specify the --force option if you wish to overwrite the existing file at "
    // + outputFile.getAbsolutePath());
    // System.exit(1);
    // }
    System.out.println("Reading connector configuration from " + configFile.getAbsoluteFile());
    final ConnectorConfig config = ConnectorConfig.from(configFile);
    backup(config, outputFile);
}

Example 5 with ConnectorConfig

use of com.couchbase.connector.config.es.ConnectorConfig in project couchbase-elasticsearch-connector by couchbase.

the class ElasticsearchConnector method main.

public static void main(String... args) throws Throwable {
    LOGGER.info("Couchbase Elasticsearch Connector version {}", getVersionString());
    final OptionsParser parser = new OptionsParser();
    final OptionSet options = parser.parse(args);
    final File configFile = options.valueOf(parser.configFile);
    System.out.println("Reading connector configuration from " + configFile.getAbsoluteFile());
    ConnectorConfig config = ConnectorConfig.from(configFile);
    final PanicButton panicButton = new DefaultPanicButton();
    boolean watchK8sReplicas = "true".equals(System.getenv("CBES_K8S_WATCH_REPLICAS"));
    boolean getMemberNumberFromHostname = watchK8sReplicas || "true".equals(System.getenv("CBES_K8S_STATEFUL_SET"));
    if (getMemberNumberFromHostname) {
        int memberNumber = StatefulSetInfo.fromHostname().podOrdinal + 1;
        LOGGER.info("Getting group member number from Kubernetes pod hostname: {}", memberNumber);
        // This is a kludge. The Membership class validates its arguments, so you can't have a Membership
        // of "4 of 1", for example. If we plan to get the group size from the Kubernetes StatefulSet,
        // bypass this validation by temporarily setting the group size to the largest sane value (1024).
        // We'll dial it down to the actual size of the StatefulSet a bit later on.
        int clusterSize = watchK8sReplicas ? 1024 : config.group().staticMembership().getClusterSize();
        config = transformMembership(config, m -> Membership.of(memberNumber, clusterSize));
    }
    KubernetesClient k8sClient = null;
    try {
        if (watchK8sReplicas) {
            k8sClient = new DefaultKubernetesClient();
            LOGGER.info("Activating native Kubernetes integration; connector will use StatefulSet spec" + " to determine group size." + " This mode requires a Kubernetes service account with 'get' and 'watch', and 'list'" + " permissions for the StatefulSet.");
            int k8sReplicas = ReplicaChangeWatcher.getReplicasAndPanicOnChange(k8sClient, panicButton);
            config = transformMembership(config, m -> Membership.of(m.getMemberNumber(), k8sReplicas));
        }
        if (watchK8sReplicas || getMemberNumberFromHostname) {
            LOGGER.info("Patched configuration with info from Kubernetes environment; membership = {}", config.group().staticMembership());
        }
        if (config.group().staticMembership().getClusterSize() > 1024) {
            panicButton.panic("Invalid group size configuration; totalMembers must be <= 1024." + " Did you forget to set the CBES_TOTAL_MEMBERS environment variable?");
        }
        Duration startupQuietPeriod = watchK8sReplicas ? ReplicaChangeWatcher.startupQuietPeriod() : Duration.ZERO;
        run(config, panicButton, startupQuietPeriod);
    } finally {
        if (k8sClient != null) {
            // so client threads don't prevent app from exiting
            k8sClient.close();
        }
    }
}