Examples with DataStaxBulkLoader com.datastax.oss.dsbulk.runner.DataStaxBulkLoader used on opensource projects

Example 61 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method error_load_missing_field.

@Test
void error_load_missing_field() throws Exception {
    SimulacronUtils.primeTables(simulacron, new Keyspace("ks1", new Table("table1", new Column("a", INT), new Column("b", TEXT), new Column("c", BOOLEAN), new Column("d", INT))));
    String[] args = { "load", "--log.maxErrors", "2", "--log.verbosity", "2", "-header", "true", "--connector.csv.url", StringUtils.quoteJson(getClass().getResource("/missing-extra.csv")), "--schema.query", "INSERT INTO ks1.table1 (a, b, c, d) VALUES (:a, :b, :c, :d)", "--schema.mapping", "A = a, B = b, C = c, D = d", "--schema.allowMissingFields", "false" };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_TOO_MANY_ERRORS);
    assertThat(logs.getAllMessagesAsString()).contains("aborted: Too many errors, the maximum allowed is 2").contains("Records: total: 3, successful: 0, failed: 3");
    validateNumberOfBadRecords(3);
    validateExceptionsLog(3, "Required field D (mapped to column d) was missing from record", "mapping-errors.log");
}

Example 62 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method unload_failure_during_read_single_thread.

@Test
void unload_failure_during_read_single_thread() {
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithError(SELECT_FROM_IP_BY_COUNTRY, new SyntaxErrorResult("Invalid table", 0L, false));
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "-header", "false", "--connector.csv.url", quoteJson(unloadDir), "--connector.csv.maxConcurrentFiles", "1", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_FATAL_ERROR);
    validateQueryCount(simulacron, 0, SELECT_FROM_IP_BY_COUNTRY, LOCAL_ONE);
    validatePrepare(simulacron, SELECT_FROM_IP_BY_COUNTRY);
}

Example 63 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method error_load_extra_field.

@Test
void error_load_extra_field() throws Exception {
    SimulacronUtils.primeTables(simulacron, new Keyspace("ks1", new Table("table1", new Column("a", INT), new Column("b", TEXT))));
    String[] args = { "load", "--log.maxErrors", "2", "--log.verbosity", "2", "-header", "true", "--connector.csv.url", StringUtils.quoteJson(getClass().getResource("/missing-extra.csv")), "--schema.query", "INSERT INTO ks1.table1 (a, b) VALUES (:a, :b)", "--schema.mapping", "A = a, B = b", "--schema.allowExtraFields", "false" };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_TOO_MANY_ERRORS);
    assertThat(logs.getAllMessagesAsString()).contains("aborted: Too many errors, the maximum allowed is 2").contains("Records: total: 3, successful: 0, failed: 3");
    validateNumberOfBadRecords(3);
    validateExceptionsLog(3, "Extraneous field C was found in record", "mapping-errors.log");
}

Example 64 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method error_load_primary_key_cannot_be_null.

@Test
void error_load_primary_key_cannot_be_null() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "--log.maxErrors", "9", "--log.verbosity", "2", "-header", "false", "--connector.csv.url", StringUtils.quoteJson(getClass().getResource("/ip-by-country-pk-null.csv")), "--codec.nullStrings", "[NULL]", "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_TOO_MANY_ERRORS);
    assertThat(logs.getAllMessagesAsString()).contains("aborted: Too many errors, the maximum allowed is 9").contains("Records: total: 24, successful: 14, failed: 10");
    // the number of writes may vary due to the abortion
    validateNumberOfBadRecords(10);
    validateExceptionsLog(10, "Primary key column country_code cannot be set to null", "mapping-errors.log");
}

Example 65 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method full_unload_large_result_set.

@Test
void full_unload_large_result_set() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithResultSet(SELECT_FROM_IP_BY_COUNTRY, 10_000);
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "-header", "false", "--connector.csv.url", quoteJson(unloadDir), "--connector.csv.maxConcurrentFiles", "1C", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateQueryCount(simulacron, 1, SELECT_FROM_IP_BY_COUNTRY, ConsistencyLevel.LOCAL_ONE);
    validateOutputFiles(10_000, unloadDir);
}