Examples with ExitStatus com.datastax.oss.dsbulk.runner.ExitStatus used on opensource projects

Example 1 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method load_ttl_timestamp_now_in_query_and_mapping_real_names.

@Test
void load_ttl_timestamp_now_in_query_and_mapping_real_names() {
    session.execute("DROP TABLE IF EXISTS table_ttl_timestamp");
    session.execute("CREATE TABLE table_ttl_timestamp (key int PRIMARY KEY, value text, loaded_at timeuuid)");
    List<String> args = Lists.newArrayList("load", "--log.directory", quoteJson(logDir), "--connector.csv.ignoreLeadingWhitespaces", "true", "--connector.csv.ignoreTrailingWhitespaces", "true", "--connector.csv.url", ClassLoader.getSystemResource("ttl-timestamp.csv").toExternalForm(), "--driver.pooling.local.connections", "1", "--schema.keyspace", session.getKeyspace().get().asInternal(), "--schema.query", "insert into table_ttl_timestamp (key, value, loaded_at) values (:key, :value, now()) using ttl :t1 and timestamp :t2", "--schema.mapping", "*=*, created_at = t2, time_to_live = t1");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertTTLAndTimestamp();
}

Example 2 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method unload_user_defined_functions_custom_query.

@Test
void unload_user_defined_functions_custom_query() throws IOException {
    assumeTrue(ccm.getCassandraVersion().compareTo(V2_2) >= 0, "User-defined functions are not compatible with C* < 2.2");
    session.execute("DROP TABLE IF EXISTS udf_table");
    session.execute("CREATE TABLE udf_table (pk int PRIMARY KEY, \"Value 1\" int, \"Value 2\" int)");
    session.execute("INSERT INTO udf_table (pk, \"Value 1\", \"Value 2\") VALUES (0,1,2)");
    session.execute("DROP FUNCTION IF EXISTS plus");
    session.execute("CREATE FUNCTION plus(s int, v int) RETURNS NULL ON NULL INPUT RETURNS int LANGUAGE java AS 'return s+v;';");
    List<String> args = Lists.newArrayList("unload", "--log.directory", quoteJson(logDir), "-header", "true", "--connector.csv.url", quoteJson(unloadDir), "--connector.csv.maxConcurrentFiles", "1", "--schema.keyspace", session.getKeyspace().get().asInternal(), "--schema.query", quoteJson("SELECT " + "\"Value 1\", \"Value 2\", " + "plus(\"Value 1\", \"Value 2\") AS \"SUM\"" + "FROM udf_table"));
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    List<String> lines = FileUtils.readAllLinesInDirectoryAsStream(unloadDir).collect(Collectors.toList());
    assertThat(lines).containsExactly("Value 1,Value 2,SUM", "1,2,3");
}

Example 3 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method full_unload_text_truncation.

/**
 * Test for DAT-400.
 */
@Test
void full_unload_text_truncation() throws Exception {
    session.execute("CREATE TABLE IF NOT EXISTS test_truncation (" + "id text PRIMARY KEY," + "text_column text," + "set_text_column set<text>," + "list_text_column list<text>," + "map_text_column map<text, text>)");
    session.execute("insert into test_truncation (id, text_column) values ('test1', 'this is text')");
    session.execute("insert into test_truncation (id, text_column) values ('test2', '1234 this text started with a number')");
    session.execute("insert into test_truncation (id, text_column) values ('test3', 'this text ended with a number 1234')");
    session.execute("insert into test_truncation (id, text_column) values ('test4', 'this text is 1234 with a number')");
    session.execute("insert into test_truncation (id, text_column) values ('test5', '1234startswithanumbernospaces')");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234 test text'} where id='test6'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234 test text'} where id='test7'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234 test text', 'this starts with text'} where id='test7'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'this starts with text'} where id='test8'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234thisisnospaces'} where id='test9'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'122 more text'} where id='test9'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'122 more text'} where id='test10'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'8595 more text'} where id='test10'");
    session.execute("update test_truncation set map_text_column = {'1234 test text': '789 value text'} where id='test11'");
    session.execute("update test_truncation set list_text_column = ['1234 test text', '789 value text'] where id='test12'");
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.csv.url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.csv.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("test_truncation");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(FileUtils.readAllLinesInDirectoryAsStreamExcludingHeaders(unloadDir)).containsExactlyInAnyOrder("test1,[],{},[],this is text", "test2,[],{},[],1234 this text started with a number", "test3,[],{},[],this text ended with a number 1234", "test4,[],{},[],this text is 1234 with a number", "test5,[],{},[],1234startswithanumbernospaces", "test6,[],{},\"[\\\"1234 test text\\\"]\",", "test7,[],{},\"[\\\"1234 test text\\\",\\\"this starts with text\\\"]\",", "test8,[],{},\"[\\\"this starts with text\\\"]\",", "test9,[],{},\"[\\\"122 more text\\\",\\\"1234thisisnospaces\\\"]\",", "test10,[],{},\"[\\\"122 more text\\\",\\\"8595 more text\\\"]\",", "test11,[],\"{\\\"1234 test text\\\":\\\"789 value text\\\"}\",[],", "test12,\"[\\\"1234 test text\\\",\\\"789 value text\\\"]\",{},[],");
}

Example 4 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method full_load_query_warnings.

/**
 * Test for DAT-451.
 */
@Test
void full_load_query_warnings() throws Exception {
    assumeTrue(ccm.getCassandraVersion().compareTo(V3) >= 0, "Query warnings are only present in C* >= 3.0");
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--log.maxQueryWarnings");
    args.add("1");
    args.add("--connector.csv.url");
    args.add(quoteJson(CsvUtils.CSV_RECORDS));
    args.add("--connector.csv.header");
    args.add("true");
    args.add("--batch.mode");
    args.add("REPLICA_SET");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_INDEXED);
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(500, "SELECT * FROM ip_by_country");
    validatePositionsFile(CsvUtils.CSV_RECORDS, 500);
    /*
    Unlogged batch covering N partitions detected against table [ks1.ip_by_country].
    You should use a logged batch for atomicity, or asynchronous writes for performance.
    DSE 6.0+:
    Unlogged batch covering 20 partitions detected against table {ks1.ip_by_country}.
    You should use a logged batch for atomicity, or asynchronous writes for performance.
     */
    assertThat(logs).hasMessageContaining("Query generated server-side warning").hasMessageMatching("Unlogged batch covering \\d+ partitions detected").hasMessageContaining(session.getKeyspace().get().asCql(true) + ".ip_by_country").hasMessageContaining("The maximum number of logged query warnings has been exceeded (1); " + "subsequent warnings will not be logged.");
}

Example 5 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method duplicate_mappings.

/**
 * Test for DAT-373.
 */
@Test
void duplicate_mappings() throws IOException {
    session.execute("DROP TABLE IF EXISTS dat373");
    session.execute("CREATE TABLE dat373 (pk int PRIMARY KEY, v1 int, v2 int)");
    List<String> args = Lists.newArrayList("load", "--log.directory", quoteJson(logDir), "-header", "true", "--connector.csv.url", quoteJson(getClass().getResource("/duplicates.csv")), "--schema.keyspace", session.getKeyspace().get().asInternal(), "--schema.table", "dat373", "--schema.mapping", "*=*, v = v1, v = v2");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    Row row = session.execute("SELECT * FROM dat373").one();
    assertThat(row.getInt("pk")).isOne();
    assertThat(row.getInt("v1")).isEqualTo(42);
    assertThat(row.getInt("v2")).isEqualTo(42);
    args = Lists.newArrayList("unload", "--log.directory", quoteJson(logDir), "-header", "true", "--connector.csv.url", quoteJson(unloadDir), "--connector.csv.maxConcurrentFiles", "1", "--schema.keyspace", session.getKeyspace().get().asInternal(), "--schema.table", "dat373", "--schema.mapping", "pk = pk, a = v1, b = v1, c = v2, d = v2");
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    List<String> lines = FileUtils.readAllLinesInDirectoryAsStream(unloadDir).collect(Collectors.toList());
    assertThat(lines).containsExactly("pk,a,b,c,d", "1,42,42,42,42");
}