Examples with DataStaxBulkLoader com.datastax.oss.dsbulk.runner.DataStaxBulkLoader used on opensource projects

Example 76 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonConnectorEndToEndCCMIT method full_unload_text_truncation.

/**
 * Test for DAT-400.
 */
@Test
void full_unload_text_truncation() throws Exception {
    session.execute("CREATE TABLE IF NOT EXISTS test_truncation (" + "id text PRIMARY KEY," + "text_column text," + "set_text_column set<text>," + "list_text_column list<text>," + "map_text_column map<text, text>)");
    session.execute("insert into test_truncation (id, text_column) values ('test1', 'this is text')");
    session.execute("insert into test_truncation (id, text_column) values ('test2', '1234 this text started with a number')");
    session.execute("insert into test_truncation (id, text_column) values ('test3', 'this text ended with a number 1234')");
    session.execute("insert into test_truncation (id, text_column) values ('test4', 'this text is 1234 with a number')");
    session.execute("insert into test_truncation (id, text_column) values ('test5', '1234startswithanumbernospaces')");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234 test text'} where id='test6'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234 test text'} where id='test7'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234 test text', 'this starts with text'} where id='test7'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'this starts with text'} where id='test8'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'1234thisisnospaces'} where id='test9'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'122 more text'} where id='test9'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'122 more text'} where id='test10'");
    session.execute("update test_truncation set set_text_column = set_text_column + {'8595 more text'} where id='test10'");
    session.execute("update test_truncation set map_text_column = {'1234 test text': '789 value text'} where id='test11'");
    session.execute("update test_truncation set list_text_column = ['1234 test text', '789 value text'] where id='test12'");
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("json");
    args.add("--connector.json.url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.json.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("test_truncation");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    Assertions.assertThat(FileUtils.readAllLinesInDirectoryAsStream(unloadDir)).containsExactlyInAnyOrder("{\"id\":\"test1\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[],\"text_column\":\"this is text\"}", "{\"id\":\"test2\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[],\"text_column\":\"1234 this text started with a number\"}", "{\"id\":\"test3\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[],\"text_column\":\"this text ended with a number 1234\"}", "{\"id\":\"test4\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[],\"text_column\":\"this text is 1234 with a number\"}", "{\"id\":\"test5\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[],\"text_column\":\"1234startswithanumbernospaces\"}", "{\"id\":\"test6\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[\"1234 test text\"],\"text_column\":null}", "{\"id\":\"test7\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[\"1234 test text\",\"this starts with text\"],\"text_column\":null}", "{\"id\":\"test8\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[\"this starts with text\"],\"text_column\":null}", "{\"id\":\"test9\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[\"122 more text\",\"1234thisisnospaces\"],\"text_column\":null}", "{\"id\":\"test10\",\"list_text_column\":[],\"map_text_column\":{},\"set_text_column\":[\"122 more text\",\"8595 more text\"],\"text_column\":null}", "{\"id\":\"test11\",\"list_text_column\":[],\"map_text_column\":{\"1234 test text\":\"789 value text\"},\"set_text_column\":[],\"text_column\":null}", "{\"id\":\"test12\",\"list_text_column\":[\"1234 test text\",\"789 value text\"],\"map_text_column\":{},\"set_text_column\":[],\"text_column\":null}");
}

Example 77 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonConnectorEndToEndCCMIT method full_load_unload_with_spaces.

/**
 * Attempt to load and unload data using ccm for a keyspace and table that is case-sensitive, and
 * with a column name containing spaces. The source data also has a header row containing spaces,
 * and the source data contains a multi-line value.
 */
@Test
void full_load_unload_with_spaces() throws Exception {
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("json");
    args.add("-url");
    args.add(StringUtils.quoteJson(JsonUtils.JSON_RECORDS_WITH_SPACES));
    args.add("--schema.mapping");
    args.add(StringUtils.quoteJson("key=key,\"my source\"=\"my destination\""));
    args.add("-k");
    args.add("MYKS");
    args.add("-t");
    args.add("WITH_SPACES");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(1, "SELECT * FROM \"MYKS\".\"WITH_SPACES\"");
    FileUtils.deleteDirectory(logDir);
    args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("json");
    args.add("-url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.json.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.mapping");
    args.add(StringUtils.quoteJson("key=key,\"my source\"=\"my destination\""));
    args.add("-k");
    args.add("MYKS");
    args.add("-t");
    args.add("WITH_SPACES");
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateOutputFiles(1, unloadDir);
}

Example 78 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonConnectorEndToEndCCMIT method full_load_unload_snappy.

/**
 * Simple test case which attempts to load and unload data using ccm and compression (Snappy).
 */
@Test
void full_load_unload_snappy() throws Exception {
    assumeTrue(session.getContext().getProtocolVersion().getCode() != 5, "Snappy compression is not supported in protocol v5");
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("json");
    args.add("--driver.protocol.compression");
    args.add("SNAPPY");
    args.add("--connector.json.url");
    args.add(StringUtils.quoteJson(JsonUtils.JSON_RECORDS_UNIQUE));
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_NAMED);
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(24, "SELECT * FROM ip_by_country");
    FileUtils.deleteDirectory(logDir);
    args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("json");
    args.add("--driver.protocol.compression");
    args.add("SNAPPY");
    args.add("--connector.json.url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.json.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_NAMED);
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateOutputFiles(24, unloadDir);
}

Example 79 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonConnectorEndToEndCCMIT method load_numeric_fields.

/**
 * Test for DAT-377.
 */
@Test
void load_numeric_fields() {
    session.execute("drop table if exists numeric_fields");
    session.execute("create table numeric_fields (pk int, cc int, v int, primary key (pk, cc))");
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("json");
    args.add("--connector.json.url");
    args.add(StringUtils.quoteJson(ClassLoader.getSystemResource("numeric-fields.json")));
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("numeric_fields");
    args.add("--schema.mapping");
    args.add("0=pk,1=cc,2=v");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(1, "SELECT * FROM numeric_fields");
}

Example 80 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonConnectorEndToEndCCMIT method full_load_unload.

/**
 * Simple test case which attempts to load and unload data using ccm.
 */
@Test
void full_load_unload() throws Exception {
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("json");
    args.add("--connector.json.url");
    args.add(StringUtils.quoteJson(JsonUtils.JSON_RECORDS_UNIQUE));
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_NAMED);
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(24, "SELECT * FROM ip_by_country");
    FileUtils.deleteDirectory(logDir);
    args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("json");
    args.add("--connector.json.url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.json.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("ip_by_country");
    args.add("--schema.mapping");
    args.add(IP_BY_COUNTRY_MAPPING_NAMED);
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateOutputFiles(24, unloadDir);
}