Search in sources :

Example 51 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class SSLEncryptionEndToEndCCMIT method full_load_unload_openssl_legacy_settings.

@Test
void full_load_unload_openssl_legacy_settings() throws Exception {
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.csv.url");
    args.add(StringUtils.quoteJson(CsvUtils.CSV_RECORDS_UNIQUE));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_INDEXED);
    args.add("--driver.auth.username");
    args.add("cassandra");
    args.add("--driver.auth.password");
    args.add("cassandra");
    args.add("--driver.ssl.provider");
    args.add("OpenSSL");
    args.add("--driver.ssl.openssl.keyCertChain");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_CERT_CHAIN_FILE.toString());
    args.add("--driver.ssl.openssl.privateKey");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_PRIVATE_KEY_FILE.toString());
    args.add("--driver.ssl.truststore.path");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_FILE.toString());
    args.add("--driver.ssl.truststore.password");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_PASSWORD);
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(24, "SELECT * FROM ip_by_country");
    assertThat(logs).hasMessageContaining("completed successfully");
    assertThat(stderr.getStreamAsStringPlain()).contains("completed successfully");
    logs.clear();
    stderr.clear();
    FileUtils.deleteDirectory(logDir);
    args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.csv.url");
    args.add(StringUtils.quoteJson(unloadDir));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--connector.csv.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_INDEXED);
    args.add("--driver.auth.username");
    args.add("cassandra");
    args.add("--driver.auth.password");
    args.add("cassandra");
    args.add("--driver.ssl.provider");
    args.add("OpenSSL");
    args.add("--driver.ssl.openssl.keyCertChain");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_CERT_CHAIN_FILE.toString());
    args.add("--driver.ssl.openssl.privateKey");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_PRIVATE_KEY_FILE.toString());
    args.add("--driver.ssl.truststore.path");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_FILE.toString());
    args.add("--driver.ssl.truststore.password");
    args.add(DefaultCCMCluster.DEFAULT_CLIENT_TRUSTSTORE_PASSWORD);
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateOutputFiles(24, unloadDir);
    assertThat(logs).hasMessageContaining("completed successfully");
    assertThat(stderr.getStreamAsStringPlain()).contains("completed successfully");
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) CqlIdentifier(com.datastax.oss.driver.api.core.CqlIdentifier) Test(org.junit.jupiter.api.Test)

Example 52 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class SearchEndToEndCCMIT method normal_unload_of_search_enabled_table.

/**
 * Test for DAT-365: regular unload of a search-enabled table should not contain the solr_query
 * column.
 */
@Test
void normal_unload_of_search_enabled_table() {
    session.execute("CREATE TABLE IF NOT EXISTS test_search2 (pk int, cc int, v varchar, PRIMARY KEY (pk, cc))");
    session.execute("CREATE SEARCH INDEX IF NOT EXISTS ON test_search2 WITH COLUMNS v { indexed:true };");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 0, 'foo')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 1, 'bar')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 2, 'qix')");
    // Wait until index is built
    await().atMost(ONE_MINUTE).until(() -> !session.execute("SELECT v FROM test_search2 WHERE solr_query = '{\"q\": \"v:foo\"}'").all().isEmpty());
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("test_search2");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(records).hasSize(3).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("foo");
    }, Index.atIndex(0)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("1");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("bar");
    }, Index.atIndex(1)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("2");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("qix");
    }, Index.atIndex(2));
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) StreamCapture(com.datastax.oss.dsbulk.tests.logging.StreamCapture) BeforeEach(org.junit.jupiter.api.BeforeEach) CqlIdentifier(com.datastax.oss.driver.api.core.CqlIdentifier) CCMRequirements(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMRequirements) MockConnector(com.datastax.oss.dsbulk.runner.tests.MockConnector) CCMWorkload(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMWorkload) ArrayList(java.util.ArrayList) CCMConfig(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMConfig) CqlSession(com.datastax.oss.driver.api.core.CqlSession) EndToEndUtils.assertStatus(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.assertStatus) CCMCluster(com.datastax.oss.dsbulk.tests.ccm.CCMCluster) Tag(org.junit.jupiter.api.Tag) ONE_MINUTE(org.awaitility.Durations.ONE_MINUTE) Record(com.datastax.oss.dsbulk.connectors.api.Record) LogInterceptor(com.datastax.oss.dsbulk.tests.logging.LogInterceptor) StreamInterceptor(com.datastax.oss.dsbulk.tests.logging.StreamInterceptor) Awaitility.await(org.awaitility.Awaitility.await) TestAssertions.assertThat(com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat) StringUtils(com.datastax.oss.dsbulk.tests.utils.StringUtils) DefaultMappedField(com.datastax.oss.dsbulk.connectors.api.DefaultMappedField) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) CCMVersionRequirement(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMVersionRequirement) Workload(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Workload) LogCapture(com.datastax.oss.dsbulk.tests.logging.LogCapture) Test(org.junit.jupiter.api.Test) STATUS_OK(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_OK) List(java.util.List) Type(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type) ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) Index(org.assertj.core.data.Index) WARN(org.slf4j.event.Level.WARN) STDERR(com.datastax.oss.dsbulk.tests.logging.StreamType.STDERR) DefaultMappedField(com.datastax.oss.dsbulk.connectors.api.DefaultMappedField) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) CqlIdentifier(com.datastax.oss.driver.api.core.CqlIdentifier) Test(org.junit.jupiter.api.Test)

Example 53 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class SearchEndToEndCCMIT method full_unload_search_solr_query.

/**
 * Test for DAT-309: unload of a Solr query
 */
@Test
void full_unload_search_solr_query() {
    session.execute("CREATE TABLE IF NOT EXISTS test_search (pk int, cc int, v varchar, PRIMARY KEY (pk, cc))");
    session.execute("CREATE SEARCH INDEX IF NOT EXISTS ON test_search WITH COLUMNS v { indexed:true };");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 0, 'foo')");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 1, 'bar')");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 2, 'qix')");
    String query = "SELECT v FROM test_search WHERE solr_query = '{\"q\": \"v:foo\"}'";
    // Wait until index is built
    await().atMost(ONE_MINUTE).until(() -> !session.execute(query).all().isEmpty());
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--executor.continuousPaging.enabled");
    args.add("true");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.query");
    args.add(StringUtils.quoteJson(query));
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(logs).hasMessageContaining("completed successfully").hasMessageContaining("Continuous paging is enabled but is not compatible with search queries; disabling");
    assertThat(stderr.getStreamAsStringPlain()).contains("completed successfully").contains("Continuous paging is enabled but is not compatible with search queries; disabling");
    assertThat(records).singleElement().satisfies(record -> {
        assertThat(record.fields()).hasSize(1);
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("foo");
    });
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) DefaultMappedField(com.datastax.oss.dsbulk.connectors.api.DefaultMappedField) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) CqlIdentifier(com.datastax.oss.driver.api.core.CqlIdentifier) Test(org.junit.jupiter.api.Test)

Example 54 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class TableReadEndToEndCCMITBase method full_unload_custom_result_set.

@ParameterizedTest(name = "[{index}] unload keyspace {0} table {1} (custom result set)")
@CsvSource({ "RF_1,SINGLE_PK", "RF_1,composite_pk", "rf_2,SINGLE_PK", "rf_2,composite_pk", "rf_3,SINGLE_PK", "rf_3,composite_pk" })
void full_unload_custom_result_set(String keyspace, String table) {
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.query");
    if (table.equals("SINGLE_PK")) {
        args.add(StringUtils.quoteJson(String.format("SELECT pk, ttl(v), writetime(v), token(pk), now() FROM \"%s\".\"%s\"", keyspace, table)));
    } else {
        args.add(StringUtils.quoteJson(String.format("SELECT \"PK1\", ttl(v), writetime(v), token(\"PK1\", \"PK2\"), now() FROM \"%s\".\"%s\"", keyspace, table)));
    }
    args.add("--driver.basic.request.consistency");
    args.add("ALL");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertUnload();
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) CsvSource(org.junit.jupiter.params.provider.CsvSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 55 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class TableReadEndToEndCCMITBase method full_count_custom_where_clause.

@ParameterizedTest(name = "[{index}] count keyspace {0} table {1} (custom query)")
@ArgumentsSource(CountWorkflowCustomQueryArgumentsProvider.class)
void full_count_custom_where_clause(String keyspace, String table) {
    List<String> args = new ArrayList<>();
    args.add("count");
    args.add("-stats");
    args.add("global");
    args.add("--schema.query");
    if (table.equals("SINGLE_PK")) {
        args.add(StringUtils.quoteJson(String.format("SELECT * FROM \"%s\".\"%s\" WHERE token(pk) > %s AND token(pk) <= %s", keyspace, table, TokenUtils.getTokenValue(getMinToken()), TokenUtils.getTokenValue(getMaxToken()))));
    } else {
        args.add(StringUtils.quoteJson(String.format("SELECT * FROM \"%s\".\"%s\"  WHERE token(\"PK1\", \"PK2\") > %s AND token(\"PK1\", \"PK2\") <= %s", keyspace, table, TokenUtils.getTokenValue(getMinToken()), TokenUtils.getTokenValue(getMaxToken()))));
    }
    args.add("--driver.basic.request.consistency");
    args.add("ALL");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertCount(keyspace, table, Collections.singleton("global"));
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) ArgumentsSource(org.junit.jupiter.params.provider.ArgumentsSource)

Aggregations

DataStaxBulkLoader (com.datastax.oss.dsbulk.runner.DataStaxBulkLoader)165 ExitStatus (com.datastax.oss.dsbulk.runner.ExitStatus)165 Test (org.junit.jupiter.api.Test)142 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)135 ArrayList (java.util.ArrayList)75 Row (com.datastax.oss.driver.api.core.cql.Row)30 RequestPrime (com.datastax.oss.simulacron.common.cluster.RequestPrime)30 Prime (com.datastax.oss.simulacron.common.stubbing.Prime)30 CqlIdentifier (com.datastax.oss.driver.api.core.CqlIdentifier)22 ResultSet (com.datastax.oss.driver.api.core.cql.ResultSet)14 MethodSource (org.junit.jupiter.params.provider.MethodSource)10 Column (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils.Column)9 Table (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils.Table)9 URL (java.net.URL)9 EndToEndUtils.primeIpByCountryTable (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.primeIpByCountryTable)8 CsvSource (org.junit.jupiter.params.provider.CsvSource)7 Record (com.datastax.oss.dsbulk.connectors.api.Record)6 SimulacronUtils (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils)5 Keyspace (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils.Keyspace)5 Path (java.nio.file.Path)5