Search in sources :

Example 21 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method full_load_unload_load_of_custom_types.

/**
 * Test for customType without associated codec. Data should be inserted as the blob. To transform
 * DynamicCompositeType into blob:
 *
 * <pre>{@code
 * ByteBuffer foo = com.datastax.driver.core.TestUtils.serializeForDynamicCompositeType("foo",32);
 * String blobHex = com.datastax.driver.core.utils.Bytes.toHexString(foo.array());
 * }</pre>
 *
 * <p>and uses blobHex to insert into table custom_types_table - c1 column (see custom-type.csv
 * file for actual hex value)
 */
@Test
void full_load_unload_load_of_custom_types() throws Exception {
    URL customTypesCsv = ClassLoader.getSystemResource("custom-type.csv");
    session.execute("CREATE TABLE custom_types_table (k int PRIMARY KEY, c1 'DynamicCompositeType(s => UTF8Type, i => Int32Type)')");
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.csv.url");
    args.add(quoteJson(customTypesCsv));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("custom_types_table");
    args.add("--schema.mapping");
    args.add("k, c1");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(1, "SELECT * FROM custom_types_table");
    FileUtils.deleteDirectory(logDir);
    args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.csv.url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--connector.csv.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("custom_types_table");
    args.add("--schema.mapping");
    args.add("k, c1");
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateOutputFiles(1, unloadDir);
    args = new ArrayList<>();
    args.add("load");
    args.add("--connector.csv.url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("custom_types_table");
    args.add("--schema.mapping");
    args.add("k, c1");
    status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateResultSetSize(1, "SELECT * FROM custom_types_table");
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) URL(java.net.URL) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 22 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method load_ttl_timestamp_now_in_query.

@Test
void load_ttl_timestamp_now_in_query() {
    session.execute("DROP TABLE IF EXISTS table_ttl_timestamp");
    session.execute("CREATE TABLE table_ttl_timestamp (key int PRIMARY KEY, value text, loaded_at timeuuid)");
    List<String> args = Lists.newArrayList("load", "--log.directory", quoteJson(logDir), "--connector.csv.ignoreLeadingWhitespaces", "true", "--connector.csv.ignoreTrailingWhitespaces", "true", "--connector.csv.url", ClassLoader.getSystemResource("ttl-timestamp.csv").toExternalForm(), "--schema.keyspace", session.getKeyspace().get().asInternal(), "--schema.query", "insert into table_ttl_timestamp (key, value, loaded_at) " + "values (:key, :value, now()) " + "using ttl :time_to_live and timestamp :created_at");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertTTLAndTimestamp();
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 23 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method should_respect_mapping_variables_order.

/**
 * Test for DAT-253.
 */
@Test
void should_respect_mapping_variables_order() throws Exception {
    session.execute("DROP TABLE IF EXISTS mapping");
    session.execute("CREATE TABLE IF NOT EXISTS mapping (key int PRIMARY KEY, value varchar)");
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.csv.url");
    args.add(ClassLoader.getSystemResource("invalid-mapping.csv").toExternalForm());
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("mapping");
    args.add("--schema.mapping");
    args.add("value,key");
    ExitStatus loadStatus = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(loadStatus, STATUS_COMPLETED_WITH_ERRORS);
    assertThat(logs).hasMessageContaining("At least 1 record does not match the provided schema.mapping or schema.query");
    FileUtils.deleteDirectory(logDir);
    args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.csv.url");
    args.add(quoteJson(unloadDir));
    args.add("--connector.csv.header");
    args.add("false");
    args.add("--connector.csv.maxConcurrentFiles");
    args.add("1");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.table");
    args.add("mapping");
    args.add("--schema.mapping");
    // note that the entries are not in proper order,
    // the export should still order fields by index, so 'key,value' and not 'value,key'
    args.add("1=value,0=key");
    ExitStatus unloadStatus = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(unloadStatus, STATUS_OK);
    List<String> lines = FileUtils.readAllLinesInDirectoryAsStream(unloadDir).collect(Collectors.toList());
    assertThat(lines).hasSize(2).contains("1,ok1").contains("2,ok2");
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) ArrayList(java.util.ArrayList) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 24 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method function_mapped_to_primary_key_with_custom_query_and_positional_variables.

/**
 * Test for DAT-326.
 */
@Test
void function_mapped_to_primary_key_with_custom_query_and_positional_variables() {
    session.execute("DROP TABLE IF EXISTS dat326c");
    session.execute("CREATE TABLE IF NOT EXISTS dat326c (pk int, cc timeuuid, v int, PRIMARY KEY (pk, cc))");
    List<String> args = Lists.newArrayList("load", "--log.directory", quoteJson(logDir), "-header", "true", "--connector.csv.url", quoteJson(getClass().getResource("/function-pk.csv")), "--schema.keyspace", session.getKeyspace().get().asInternal(), "--schema.query", "INSERT INTO dat326c (pk, cc, v) VALUES (?, now(), ?)");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 25 with ExitStatus

use of com.datastax.oss.dsbulk.runner.ExitStatus in project dsbulk by datastax.

the class CSVConnectorEndToEndCCMIT method cas_load_with_errors.

/**
 * Test for CAS failures (DAT-384).
 */
@Test
void cas_load_with_errors() {
    session.execute("DROP TABLE IF EXISTS test_cas");
    session.execute("CREATE TABLE test_cas (pk int, cc int, v int, PRIMARY KEY (pk, cc))");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 1, 1)");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 2, 2)");
    session.execute("INSERT INTO test_cas (pk, cc, v) VALUES (1, 3, 3)");
    // two failed CAS records will cause the entire batch to fail
    // will fail
    Record record1Failed = RecordUtils.mappedCSV("pk", "1", "cc", "1", "v", "1");
    // will fail
    Record record2Failed = RecordUtils.mappedCSV("pk", "1", "cc", "2", "v", "2");
    Record record3NotApplied = // will not be applied
    RecordUtils.mappedCSV("pk", "1", "cc", "4", "v", "4");
    MockConnector.mockReads(record1Failed, record2Failed, record3NotApplied);
    List<String> args = new ArrayList<>();
    args.add("load");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().get().asInternal());
    args.add("--schema.query");
    args.add("INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_COMPLETED_WITH_ERRORS);
    Path bad = OperationDirectory.getCurrentOperationDirectory().map(dir -> dir.resolve("paxos.bad")).orElse(null);
    assertThat(bad).exists();
    assertThat(FileUtils.readAllLines(bad)).containsExactly(record1Failed.getSource().toString(), record2Failed.getSource().toString(), record3NotApplied.getSource().toString());
    Path errors = OperationDirectory.getCurrentOperationDirectory().map(dir -> dir.resolve("paxos-errors.log")).orElse(null);
    assertThat(errors).exists();
    assertThat(FileUtils.readAllLines(errors).collect(Collectors.joining("\n"))).contains(String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 1\n" + "    v: 1", record1Failed.getResource(), record1Failed.getPosition(), record1Failed.getSource()), String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 2\n" + "    v: 2", record2Failed.getResource(), record2Failed.getPosition(), record2Failed.getSource()), String.format("Resource: %s\n" + "    Position: %d\n" + "    Source: %s\n" + "    INSERT INTO test_cas (pk, cc, v) VALUES (:pk, :cc, :v) IF NOT EXISTS\n" + "    pk: 1\n" + "    cc: 4\n" + "    v: 4", record3NotApplied.getResource(), record3NotApplied.getPosition(), record3NotApplied.getSource()), "Failed conditional updates:", "\"[applied]\": false\npk: 1\ncc: 1\nv: 1", "\"[applied]\": false\npk: 1\ncc: 2\nv: 2");
    List<Row> rows = session.execute("SELECT v FROM test_cas WHERE pk = 1").all();
    assertThat(rows).hasSize(3);
    assertThat(rows.get(0).getInt(0)).isEqualTo(1);
    assertThat(rows.get(1).getInt(0)).isEqualTo(2);
    assertThat(rows.get(2).getInt(0)).isEqualTo(3);
}
Also used : ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) Path(java.nio.file.Path) UNNECESSARY(java.math.RoundingMode.UNNECESSARY) STDOUT(com.datastax.oss.dsbulk.tests.logging.StreamType.STDOUT) BeforeEach(org.junit.jupiter.api.BeforeEach) CodecUtils(com.datastax.oss.dsbulk.codecs.api.util.CodecUtils) Strings(com.datastax.oss.driver.shaded.guava.common.base.Strings) InstanceOfAssertFactories(org.assertj.core.api.InstanceOfAssertFactories) ZonedDateTime(java.time.ZonedDateTime) INSERT_INTO_IP_BY_COUNTRY(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.INSERT_INTO_IP_BY_COUNTRY) EndToEndUtils.validateExceptionsLog(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateExceptionsLog) InetAddress(java.net.InetAddress) AfterAll(org.junit.jupiter.api.AfterAll) BigDecimal(java.math.BigDecimal) BeforeAll(org.junit.jupiter.api.BeforeAll) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) Assumptions.assumeFalse(org.junit.jupiter.api.Assumptions.assumeFalse) Map(java.util.Map) BigInteger(java.math.BigInteger) Tag(org.junit.jupiter.api.Tag) FLOOR(java.math.RoundingMode.FLOOR) Path(java.nio.file.Path) StreamInterceptor(com.datastax.oss.dsbulk.tests.logging.StreamInterceptor) TestAssertions.assertThat(com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat) Set(java.util.Set) Splitter(com.datastax.oss.driver.shaded.guava.common.base.Splitter) Arguments(org.junit.jupiter.params.provider.Arguments) Stream(java.util.stream.Stream) STATUS_ABORTED_TOO_MANY_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_TOO_MANY_ERRORS) STDERR(com.datastax.oss.dsbulk.tests.logging.StreamType.STDERR) MICROSECONDS(java.util.concurrent.TimeUnit.MICROSECONDS) EndToEndUtils.validateNumberOfBadRecords(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateNumberOfBadRecords) CQLUtils(com.datastax.oss.dsbulk.tests.utils.CQLUtils) ImmutableSet(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSet) STATUS_ABORTED_FATAL_ERROR(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_ABORTED_FATAL_ERROR) ImmutableMap(com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap) ArrayList(java.util.ArrayList) CqlSession(com.datastax.oss.driver.api.core.CqlSession) EndToEndUtils.createIpByCountryTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryTable) Assumptions.assumeTrue(org.junit.jupiter.api.Assumptions.assumeTrue) EndToEndUtils.assertStatus(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.assertStatus) CCMCluster(com.datastax.oss.dsbulk.tests.ccm.CCMCluster) Row(com.datastax.oss.driver.api.core.cql.Row) ResultSet(com.datastax.oss.driver.api.core.cql.ResultSet) Files(java.nio.file.Files) TupleType(com.datastax.oss.driver.api.core.type.TupleType) DataType(com.datastax.oss.driver.api.core.type.DataType) IOException(java.io.IOException) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) UnknownHostException(java.net.UnknownHostException) LogCapture(com.datastax.oss.dsbulk.tests.logging.LogCapture) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) ChronoUnit(java.time.temporal.ChronoUnit) Assumptions(org.junit.jupiter.api.Assumptions) EndToEndUtils.validateOutputFiles(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validateOutputFiles) CqlDuration(com.datastax.oss.driver.api.core.data.CqlDuration) URL(java.net.URL) CsvUtils(com.datastax.oss.dsbulk.runner.tests.CsvUtils) MockConnector(com.datastax.oss.dsbulk.runner.tests.MockConnector) ByteBuffer(java.nio.ByteBuffer) STATUS_COMPLETED_WITH_ERRORS(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_COMPLETED_WITH_ERRORS) RecordUtils(com.datastax.oss.dsbulk.runner.tests.RecordUtils) Lists(com.datastax.oss.driver.shaded.guava.common.collect.Lists) Uuids(com.datastax.oss.driver.api.core.uuid.Uuids) LocalTime(java.time.LocalTime) URI(java.net.URI) Record(com.datastax.oss.dsbulk.connectors.api.Record) LogInterceptor(com.datastax.oss.dsbulk.tests.logging.LogInterceptor) RoundingMode(java.math.RoundingMode) MethodSource(org.junit.jupiter.params.provider.MethodSource) CodecRegistry(com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry) OperationDirectory(com.datastax.oss.dsbulk.workflow.api.log.OperationDirectory) IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_CASE_SENSITIVE) TupleValue(com.datastax.oss.driver.api.core.data.TupleValue) LineNumberReader(java.io.LineNumberReader) UUID(java.util.UUID) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) UdtValue(com.datastax.oss.driver.api.core.data.UdtValue) Objects(java.util.Objects) Test(org.junit.jupiter.api.Test) STATUS_OK(com.datastax.oss.dsbulk.runner.ExitStatus.STATUS_OK) DataTypes(com.datastax.oss.driver.api.core.type.DataTypes) List(java.util.List) Type(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type) ExitStatus(com.datastax.oss.dsbulk.runner.ExitStatus) LocalDate(java.time.LocalDate) TypeCodec(com.datastax.oss.driver.api.core.type.codec.TypeCodec) StreamCapture(com.datastax.oss.dsbulk.tests.logging.StreamCapture) CsvSource(org.junit.jupiter.params.provider.CsvSource) EndToEndUtils.validatePositionsFile(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.validatePositionsFile) HashMap(java.util.HashMap) Version(com.datastax.oss.driver.api.core.Version) CCMConfig(com.datastax.oss.dsbulk.tests.ccm.annotations.CCMConfig) ImmutableList(com.google.common.collect.ImmutableList) CompressedIOUtils(com.datastax.oss.dsbulk.io.CompressedIOUtils) OverflowStrategy(com.datastax.oss.dsbulk.codecs.api.util.OverflowStrategy) FileUtils(com.datastax.oss.dsbulk.tests.utils.FileUtils) EndToEndUtils.createIpByCountryCaseSensitiveTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createIpByCountryCaseSensitiveTable) StringUtils.quoteJson(com.datastax.oss.dsbulk.tests.utils.StringUtils.quoteJson) UTF_8(java.nio.charset.StandardCharsets.UTF_8) EndToEndUtils.createWithSpacesTable(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.createWithSpacesTable) OSS(com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type.OSS) DefaultProtocolVersion(com.datastax.oss.driver.api.core.DefaultProtocolVersion) UserDefinedType(com.datastax.oss.driver.api.core.type.UserDefinedType) IP_BY_COUNTRY_MAPPING_INDEXED(com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.IP_BY_COUNTRY_MAPPING_INDEXED) EPOCH(java.time.Instant.EPOCH) InputStream(java.io.InputStream) ArrayList(java.util.ArrayList) Record(com.datastax.oss.dsbulk.connectors.api.Record) DataStaxBulkLoader(com.datastax.oss.dsbulk.runner.DataStaxBulkLoader) Row(com.datastax.oss.driver.api.core.cql.Row) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Aggregations

DataStaxBulkLoader (com.datastax.oss.dsbulk.runner.DataStaxBulkLoader)165 ExitStatus (com.datastax.oss.dsbulk.runner.ExitStatus)165 Test (org.junit.jupiter.api.Test)142 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)135 ArrayList (java.util.ArrayList)75 Row (com.datastax.oss.driver.api.core.cql.Row)30 RequestPrime (com.datastax.oss.simulacron.common.cluster.RequestPrime)30 Prime (com.datastax.oss.simulacron.common.stubbing.Prime)30 CqlIdentifier (com.datastax.oss.driver.api.core.CqlIdentifier)22 ResultSet (com.datastax.oss.driver.api.core.cql.ResultSet)14 MethodSource (org.junit.jupiter.params.provider.MethodSource)10 Column (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils.Column)9 Table (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils.Table)9 URL (java.net.URL)9 EndToEndUtils.primeIpByCountryTable (com.datastax.oss.dsbulk.runner.tests.EndToEndUtils.primeIpByCountryTable)8 CsvSource (org.junit.jupiter.params.provider.CsvSource)7 Record (com.datastax.oss.dsbulk.connectors.api.Record)6 SimulacronUtils (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils)5 Keyspace (com.datastax.oss.dsbulk.tests.simulacron.SimulacronUtils.Keyspace)5 Path (java.nio.file.Path)5