Examples with RequestPrime com.datastax.oss.simulacron.common.cluster.RequestPrime used on opensource projects

Example 21 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method unload_write_error.

@Test
void unload_write_error() {
    Path file1 = unloadDir.resolve("output-000001.csv");
    Path file2 = unloadDir.resolve("output-000002.csv");
    Path file3 = unloadDir.resolve("output-000003.csv");
    Path file4 = unloadDir.resolve("output-000004.csv");
    MockConnector.setDelegate(new CSVConnector() {

        @Override
        public void configure(@NonNull Config settings, boolean read, boolean retainRecordSources) {
            settings = ConfigFactory.parseString("url = " + quoteJson(unloadDir) + ", header = false, maxConcurrentFiles = 4").withFallback(ConfigUtils.createReferenceConfig().getConfig("dsbulk.connector.csv"));
            super.configure(settings, read, retainRecordSources);
        }

        @NonNull
        @Override
        public Function<Publisher<Record>, Publisher<Record>> write() {
            // will cause the write workers to fail because the files already exist
            try {
                Files.createFile(file1);
                Files.createFile(file2);
                Files.createFile(file3);
                Files.createFile(file4);
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
            return super.write();
        }
    });
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithResultSet(SELECT_FROM_IP_BY_COUNTRY, 10);
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "--connector.name", "mock", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_FATAL_ERROR);
    assertThat(stdErr.getStreamAsString()).contains("failed").containsPattern("output-00000[1-4].csv");
    assertThat(logs.getAllMessagesAsString()).contains("failed").containsPattern("output-00000[1-4].csv");
}

Example 22 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method skip_test_load.

@Test
void skip_test_load() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "-header", "false", "--connector.csv.url", StringUtils.quoteJson(CsvUtils.CSV_RECORDS_SKIP), "--connector.csv.skipRecords", "3", "--connector.csv.maxRecords", "24", "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED, "--schema.allowMissingFields", "true" };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_COMPLETED_WITH_ERRORS);
    validateQueryCount(simulacron, 21, "INSERT INTO ip_by_country", LOCAL_ONE);
    validateNumberOfBadRecords(3);
    validateExceptionsLog(3, "Source:", "mapping-errors.log");
}

Example 23 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method unload_failure_during_read_multi_thread.

@Test
void unload_failure_during_read_multi_thread() {
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithError(SELECT_FROM_IP_BY_COUNTRY, new SyntaxErrorResult("Invalid table", 0L, false));
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "-header", "false", "--connector.csv.url", quoteJson(unloadDir), "--connector.csv.maxConcurrentFiles", "1C", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_FATAL_ERROR);
    validateQueryCount(simulacron, 0, SELECT_FROM_IP_BY_COUNTRY, LOCAL_ONE);
    validatePrepare(simulacron, SELECT_FROM_IP_BY_COUNTRY);
}

Example 24 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method validate_stdout.

@Test
void validate_stdout() {
    primeIpByCountryTable(simulacron);
    RequestPrime select = createQueryWithResultSet(SELECT_FROM_IP_BY_COUNTRY, 24);
    simulacron.prime(new Prime(select));
    String[] args = { "unload", "-header", "false", "--connector.csv.url", "-", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateQueryCount(simulacron, 1, SELECT_FROM_IP_BY_COUNTRY, LOCAL_ONE);
    assertThat(stdOut.getStreamLines()).hasSize(24);
}

Example 25 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method full_load_dry_run.

@Test
void full_load_dry_run() {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "-header", "false", "--connector.csv.url", StringUtils.quoteJson(CsvUtils.CSV_RECORDS_UNIQUE), "-dryRun", "true", "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateQueryCount(simulacron, 0, "INSERT INTO ip_by_country", ONE);
}