Examples with DataStaxBulkLoader com.datastax.oss.dsbulk.runner.DataStaxBulkLoader used on opensource projects

Example 11 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method full_load_crlf.

@Test
void full_load_crlf() {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "-c", "json", "--connector.json.url", StringUtils.quoteJson(JsonUtils.JSON_RECORDS_CRLF), "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateQueryCount(simulacron, 24, "INSERT INTO ip_by_country", LOCAL_ONE);
}

Example 12 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method partial_load.

@Test
void partial_load() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "-c", "json", "--connector.json.url", StringUtils.quoteJson(JsonUtils.JSON_RECORDS_PARTIAL_BAD), "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED, "--schema.allowMissingFields", "true" };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_COMPLETED_WITH_ERRORS);
    validateQueryCount(simulacron, 21, "INSERT INTO ip_by_country", LOCAL_ONE);
    validateNumberOfBadRecords(2);
    validateExceptionsLog(2, "Source:", "mapping-errors.log");
}

Example 13 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method full_load.

@Test
void full_load() {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "-c", "json", "--connector.json.url", StringUtils.quoteJson(JsonUtils.JSON_RECORDS_UNIQUE), "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(logs.getAllMessagesAsString()).contains("Records: total: 24, successful: 24, failed: 0").contains("Batches: total: 24, size: 1.00 mean, 1 min, 1 max").contains("Writes: total: 24, successful: 24, failed: 0");
    validateQueryCount(simulacron, 24, "INSERT INTO ip_by_country", LOCAL_ONE);
}

Example 14 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class NodeFilterEndToEndSimulacronIT method node_filter_allow_deprecated.

@ParameterizedTest
@MethodSource("nodesList")
void node_filter_allow_deprecated(Set<Integer> allow) {
    String[] args = { "load", "-c", "mock", "--batch.mode", "DISABLED", "--schema.keyspace", "ks1", "--schema.table", "table1", "-port", port, "--dsbulk.driver.policy.lbp.whiteList.hosts", getHostList(allow) };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(logs).hasMessageContaining("Setting dsbulk.driver.policy.lbp.whiteList.hosts is deprecated").hasMessageContaining("configure the driver directly using --datastax-java-driver.basic.load-balancing-policy.evaluator.allow (or -allow) instead.").hasMessageContaining("completed successfully").hasMessageContaining("Records: total: 1,000, successful: 1,000, failed: 0");
    for (int i = 0; i < 3; i++) {
        BoundNode node = simulacron.node(i);
        InetSocketAddress addr = node.inetSocketAddress();
        String hostName = Pattern.quote(addr.getAddress().getHostAddress());
        int port = addr.getPort();
        if (allow.isEmpty() || allow.contains(i)) {
            checkAllowed(node, hostName, port);
        } else {
            checkExcluded(node, hostName, port);
        }
    }
}

Example 15 with DataStaxBulkLoader

use of com.datastax.oss.dsbulk.runner.DataStaxBulkLoader in project dsbulk by datastax.

the class NodeFilterEndToEndSimulacronIT method node_filter_allow.

@ParameterizedTest
@MethodSource("nodesList")
void node_filter_allow(Set<Integer> allow) {
    String[] args = { "load", "-c", "mock", "--batch.mode", "DISABLED", "--schema.keyspace", "ks1", "--schema.table", "table1", "-port", port, "-allow", getHostList(allow) };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(logs).hasMessageContaining("completed successfully").hasMessageContaining("Records: total: 1,000, successful: 1,000, failed: 0");
    for (int i = 0; i < 3; i++) {
        BoundNode node = simulacron.node(i);
        InetSocketAddress addr = node.inetSocketAddress();
        String hostName = Pattern.quote(addr.getAddress().getHostAddress());
        int port = addr.getPort();
        if (allow.isEmpty() || allow.contains(i)) {
            checkAllowed(node, hostName, port);
        } else {
            checkExcluded(node, hostName, port);
        }
    }
}