Examples with DataStaxBulkLoader com.datastax.oss.dsbulk.runner.DataStaxBulkLoader used on opensource projects

Example 51 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class SSLEncryptionEndToEndCCMIT method full_load_unload_openssl_legacy_settings.

@Test
void full_load_unload_openssl_legacy_settings() throws Exception {
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.csv.url");
    args.add(StringUtils.quoteJson(CsvUtils.CSV_RECORDS_UNIQUE));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_INDEXED);
    args.add("--driver.auth.username");
    args.add("cassandra");
    args.add("--driver.auth.password");
    args.add("cassandra");
    args.add("--driver.ssl.provider");
    args.add("OpenSSL");
    args.add("--driver.ssl.openssl.keyCertChain");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_CERT_CHAIN_FILE.toString());
    args.add("--driver.ssl.openssl.privateKey");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_PRIVATE_KEY_FILE.toString());
    args.add("--driver.ssl.truststore.path");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_FILE.toString());
    args.add("--driver.ssl.truststore.password");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_PASSWORD);
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(24, "SELECT * FROM ip_by_country");
    assertThat(logs).hasMessageContaining("completed successfully");
    assertThat(stderr.getStreamAsStringPlain()).contains("completed successfully");
    logs.clear();
    stderr.clear();
    FileUtils.deleteDirectory(logDir);
    args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.csv.url");
    args.add(StringUtils.quoteJson(unloadDir));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--connector.csv.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_INDEXED);
    args.add("--driver.auth.username");
    args.add("cassandra");
    args.add("--driver.auth.password");
    args.add("cassandra");
    args.add("--driver.ssl.provider");
    args.add("OpenSSL");
    args.add("--driver.ssl.openssl.keyCertChain");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_CERT_CHAIN_FILE.toString());
    args.add("--driver.ssl.openssl.privateKey");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_PRIVATE_KEY_FILE.toString());
    args.add("--driver.ssl.truststore.path");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_FILE.toString());
    args.add("--driver.ssl.truststore.password");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_PASSWORD);
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateOutputFiles(24, unloadDir);
    assertThat(logs).hasMessageContaining("completed successfully");
    assertThat(stderr.getStreamAsStringPlain()).contains("completed successfully");
}

Example 52 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class SearchEndToEndCCMIT method normal_unload_of_search_enabled_table.

/**
 * Test for DAT-365: regular unload of a search-enabled table should not contain the solr_query
 * column.
 */
@Test
void normal_unload_of_search_enabled_table() {
    session.execute("CREATE TABLE IF NOT EXISTS test_search2 (pk int, cc int, v varchar, PRIMARY KEY (pk, cc))");
    session.execute("CREATE SEARCH INDEX IF NOT EXISTS ON test_search2 WITH COLUMNS v { indexed:true };");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 0, 'foo')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 1, 'bar')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 2, 'qix')");
    // Wait until index is built
    await().atMost(ONE_MINUTE).until(() -> !session.execute("SELECT v FROM test_search2 WHERE solr_query = '{\"q\": \"v:foo\"}'").all().isEmpty());
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("test_search2");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(records).hasSize(3).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("foo");
    }, Index.atIndex(0)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("1");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("bar");
    }, Index.atIndex(1)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("2");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("qix");
    }, Index.atIndex(2));
}

Example 53 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class SearchEndToEndCCMIT method full_unload_search_solr_query.

/**
 * Test for DAT-309: unload of a Solr query
 */
@Test
void full_unload_search_solr_query() {
    session.execute("CREATE TABLE IF NOT EXISTS test_search (pk int, cc int, v varchar, PRIMARY KEY (pk, cc))");
    session.execute("CREATE SEARCH INDEX IF NOT EXISTS ON test_search WITH COLUMNS v { indexed:true };");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 0, 'foo')");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 1, 'bar')");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 2, 'qix')");
    String query = "SELECT v FROM test_search WHERE solr_query = '{\"q\": \"v:foo\"}'";
    // Wait until index is built
    await().atMost(ONE_MINUTE).until(() -> !session.execute(query).all().isEmpty());
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--executor.continuousPaging.enabled");
    args.add("true");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.query");
    args.add(StringUtils.quoteJson(query));
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(logs).hasMessageContaining("completed successfully").hasMessageContaining("Continuous paging is enabled but is not compatible with search queries; disabling");
    assertThat(stderr.getStreamAsStringPlain()).contains("completed successfully").contains("Continuous paging is enabled but is not compatible with search queries; disabling");
    assertThat(records).singleElement().satisfies(record -> {
        assertThat(record.fields()).hasSize(1);
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("foo");
    });
}

Example 54 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class TableReadEndToEndCCMITBase method full_unload_custom_result_set.

@ParameterizedTest(name = "[{index}] unload keyspace {0} table {1} (custom result set)")
@CsvSource({ "RF_1,SINGLE_PK", "RF_1,composite_pk", "rf_2,SINGLE_PK", "rf_2,composite_pk", "rf_3,SINGLE_PK", "rf_3,composite_pk" })
void full_unload_custom_result_set(String keyspace, String table) {
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.query");
    if (table.equals("SINGLE_PK")) {
        args.add(StringUtils.quoteJson(String.format("SELECT pk, ttl(v), writetime(v), token(pk), now() FROM \"%s\".\"%s\"", keyspace, table)));
    } else {
        args.add(StringUtils.quoteJson(String.format("SELECT \"PK1\", ttl(v), writetime(v), token(\"PK1\", \"PK2\"), now() FROM \"%s\".\"%s\"", keyspace, table)));
    }
    args.add("--driver.basic.request.consistency");
    args.add("ALL");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertUnload();
}

Example 55 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class TableReadEndToEndCCMITBase method full_count_custom_where_clause.

@ParameterizedTest(name = "[{index}] count keyspace {0} table {1} (custom query)")
@ArgumentsSource(CountWorkflowCustomQueryArgumentsProvider.class)
void full_count_custom_where_clause(String keyspace, String table) {
    List<String> args = new ArrayList<>();
    args.add("count");
    args.add("-stats");
    args.add("global");
    args.add("--schema.query");
    if (table.equals("SINGLE_PK")) {
        args.add(StringUtils.quoteJson(String.format("SELECT * FROM \"%s\".\"%s\" WHERE token(pk) > %s AND token(pk) <= %s", keyspace, table, TokenUtils.getTokenValue(getMinToken()), TokenUtils.getTokenValue(getMaxToken()))));
    } else {
        args.add(StringUtils.quoteJson(String.format("SELECT * FROM \"%s\".\"%s\"  WHERE token(\"PK1\", \"PK2\") > %s AND token(\"PK1\", \"PK2\") <= %s", keyspace, table, TokenUtils.getTokenValue(getMinToken()), TokenUtils.getTokenValue(getMaxToken()))));
    }
    args.add("--driver.basic.request.consistency");
    args.add("ALL");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertCount(keyspace, table, Collections.singleton("global"));
}