Examples with RequestPrime com.datastax.oss.simulacron.common.cluster.RequestPrime used on opensource projects

Example 31 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method full_unload_large_result_set.

@Test
void full_unload_large_result_set() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithResultSet(SELECT_FROM_IP_BY_COUNTRY, 10_000);
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "-c", "json", "--connector.json.url", StringUtils.quoteJson(unloadDir), "--connector.json.maxConcurrentFiles", "1C", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateQueryCount(simulacron, 1, SELECT_FROM_IP_BY_COUNTRY, LOCAL_ONE);
    validateOutputFiles(10_000, unloadDir);
}

Example 32 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method full_unload.

@Test
void full_unload() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithResultSet(SELECT_FROM_IP_BY_COUNTRY, 24);
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "-c", "json", "--connector.json.url", StringUtils.quoteJson(unloadDir), "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(logs.getAllMessagesAsString()).contains("Records: total: 24, successful: 24, failed: 0").contains("Reads: total: 24, successful: 24, failed: 0");
    validateQueryCount(simulacron, 1, SELECT_FROM_IP_BY_COUNTRY, LOCAL_ONE);
    validateOutputFiles(24, unloadDir);
}

Example 33 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method unload_write_error.

@Test
void unload_write_error() {
    Path file1 = unloadDir.resolve("output-000001.json");
    Path file2 = unloadDir.resolve("output-000002.json");
    Path file3 = unloadDir.resolve("output-000003.json");
    Path file4 = unloadDir.resolve("output-000004.json");
    MockConnector.setDelegate(new JsonConnector() {

        @Override
        public void configure(@NonNull Config settings, boolean read, boolean retainRecordSources) {
            settings = ConfigFactory.parseString("url = " + StringUtils.quoteJson(unloadDir) + ", maxConcurrentFiles = 4").withFallback(ConfigUtils.createReferenceConfig().getConfig("dsbulk.connector.json"));
            super.configure(settings, read, retainRecordSources);
        }

        @NonNull
        @Override
        public Function<Publisher<Record>, Publisher<Record>> write() {
            // will cause the write workers to fail because the files already exist
            try {
                Files.createFile(file1);
                Files.createFile(file2);
                Files.createFile(file3);
                Files.createFile(file4);
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
            return super.write();
        }

        @Override
        public boolean supports(@NonNull ConnectorFeature feature) {
            return true;
        }
    });
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithResultSet(SELECT_FROM_IP_BY_COUNTRY, 10);
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "--connector.name", "mock", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_FATAL_ERROR);
    assertThat(stdErr.getStreamAsString()).contains("failed").containsPattern("output-00000[1-4].json");
    assertThat(logs.getAllMessagesAsString()).contains("failed").containsPattern("output-00000[1-4].json");
}

Example 34 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method unload_failure_during_read_single_thread.

@Test
void unload_failure_during_read_single_thread() {
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithError(SELECT_FROM_IP_BY_COUNTRY, new SyntaxErrorResult("Invalid table", 0L, false));
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "-c", "json", "--connector.json.url", StringUtils.quoteJson(unloadDir), "--connector.json.maxConcurrentFiles", "1", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_FATAL_ERROR);
    validateQueryCount(simulacron, 0, SELECT_FROM_IP_BY_COUNTRY, LOCAL_ONE);
    validatePrepare(simulacron, SELECT_FROM_IP_BY_COUNTRY);
}

Example 35 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class EndToEndUtils method createQueryWithResultSet.

public static RequestPrime createQueryWithResultSet(String query, int numOfResults) {
    Query when = new Query(query);
    LinkedHashMap<String, String> columnTypes = new LinkedHashMap<>();
    columnTypes.put("country_code", "varchar");
    columnTypes.put("country_name", "varchar");
    columnTypes.put("beginning_ip_address", "inet");
    columnTypes.put("ending_ip_address", "inet");
    columnTypes.put("beginning_ip_number", "bigint");
    columnTypes.put("ending_ip_number", "bigint");
    List<LinkedHashMap<String, Object>> rows = new ArrayList<>();
    for (int i = 0; i < numOfResults; i++) {
        LinkedHashMap<String, Object> row = new LinkedHashMap<>();
        row.put("country_code", "country" + i);
        row.put("country_name", "country" + i);
        row.put("beginning_ip_address", "127.0.0.1");
        row.put("ending_ip_address", "127.2.0.1");
        row.put("beginning_ip_number", Integer.toString(i));
        row.put("ending_ip_number", Integer.toString(i));
        rows.add(row);
    }
    SuccessResult then = new SuccessResult(rows, columnTypes);
    return new RequestPrime(when, then);
}