Examples with RequestPrime com.datastax.oss.simulacron.common.cluster.RequestPrime used on opensource projects

Example 11 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method error_load_primary_key_cannot_be_null.

@Test
void error_load_primary_key_cannot_be_null() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "--log.maxErrors", "9", "--log.verbosity", "2", "-header", "false", "--connector.csv.url", StringUtils.quoteJson(getClass().getResource("/ip-by-country-pk-null.csv")), "--codec.nullStrings", "[NULL]", "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_ABORTED_TOO_MANY_ERRORS);
    assertThat(logs.getAllMessagesAsString()).contains("aborted: Too many errors, the maximum allowed is 9").contains("Records: total: 24, successful: 14, failed: 10");
    // the number of writes may vary due to the abortion
    validateNumberOfBadRecords(10);
    validateExceptionsLog(10, "Primary key column country_code cannot be set to null", "mapping-errors.log");
}

Example 12 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class CSVEndToEndSimulacronIT method full_unload_large_result_set.

@Test
void full_unload_large_result_set() throws Exception {
    primeIpByCountryTable(simulacron);
    RequestPrime prime = createQueryWithResultSet(SELECT_FROM_IP_BY_COUNTRY, 10_000);
    simulacron.prime(new Prime(prime));
    String[] args = { "unload", "-header", "false", "--connector.csv.url", quoteJson(unloadDir), "--connector.csv.maxConcurrentFiles", "1C", "--schema.keyspace", "ks1", "--schema.query", SELECT_FROM_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_INDEXED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateQueryCount(simulacron, 1, SELECT_FROM_IP_BY_COUNTRY, ConsistencyLevel.LOCAL_ONE);
    validateOutputFiles(10_000, unloadDir);
}

Example 13 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class JsonEndToEndSimulacronIT method full_load_dry_run.

@Test
void full_load_dry_run() {
    primeIpByCountryTable(simulacron);
    RequestPrime insert = createSimpleParameterizedQuery(INSERT_INTO_IP_BY_COUNTRY);
    simulacron.prime(new Prime(insert));
    String[] args = { "load", "-c", "json", "--connector.json.url", StringUtils.quoteJson(JsonUtils.JSON_RECORDS_UNIQUE), "-dryRun", "true", "--schema.keyspace", "ks1", "--schema.query", INSERT_INTO_IP_BY_COUNTRY, "--schema.mapping", IP_BY_COUNTRY_MAPPING_NAMED };
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    validateQueryCount(simulacron, 0, "INSERT INTO ip_by_country", ONE);
}

Example 14 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class SimulacronUtils method primeSystemPeersV2.

public static void primeSystemPeersV2(BoundCluster simulacron) {
    Query whenSelectSystemPeersV2 = new Query(SELECT_SYSTEM_PEERS_V2);
    ErrorResult thenThrowServerError = new ServerErrorResult("Unknown keyspace/cf pair (system.peers_v2)");
    RequestPrime primeSystemPeersV2 = new RequestPrime(whenSelectSystemPeersV2, thenThrowServerError);
    simulacron.prime(new Prime(primeSystemPeersV2));
}

Example 15 with RequestPrime

use of com.datastax.oss.simulacron.common.cluster.RequestPrime in project dsbulk by datastax.

the class SimulacronUtils method primeTables.

public static void primeTables(BoundCluster simulacron, Keyspace... keyspaces) {
    List<LinkedHashMap<String, Object>> allKeyspacesRows = new ArrayList<>();
    List<LinkedHashMap<String, Object>> allTablesRows = new ArrayList<>();
    List<LinkedHashMap<String, Object>> allColumnsRows = new ArrayList<>();
    for (Keyspace keyspace : keyspaces) {
        LinkedHashMap<String, Object> keyspaceRow = new LinkedHashMap<>();
        keyspaceRow.put("keyspace_name", keyspace.name);
        keyspaceRow.put("durable_writes", true);
        keyspaceRow.put("replication", ImmutableMap.of("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1"));
        allKeyspacesRows.add(keyspaceRow);
        Query whenSelectKeyspace = new Query(SELECT_KEYSPACES + " WHERE keyspace_name = '" + keyspace.name + '\'');
        SuccessResult thenReturnKeyspace = new SuccessResult(Collections.singletonList(keyspaceRow), new LinkedHashMap<>(KEYSPACE_COLUMNS));
        RequestPrime primeKeyspace = new RequestPrime(whenSelectKeyspace, thenReturnKeyspace);
        simulacron.prime(new Prime(primeKeyspace));
        for (Table table : keyspace.tables) {
            LinkedHashMap<String, Object> tableRow = new LinkedHashMap<>();
            tableRow.put("keyspace_name", keyspace.name);
            tableRow.put("table_name", table.name);
            tableRow.put("bloom_filter_fp_chance", 0.01d);
            tableRow.put("caching", ImmutableMap.of("keys", "ALL", "rows_per_partition", "NONE"));
            tableRow.put("cdc", null);
            tableRow.put("comment", "");
            tableRow.put("compaction", ImmutableMap.of("class", "org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy", "max_threshold", "32", "min_threshold", "4"));
            tableRow.put("compression", ImmutableMap.of("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor"));
            tableRow.put("crc_check_chance", 1d);
            tableRow.put("dclocal_read_repair_chance", 0.1d);
            tableRow.put("default_time_to_live", 0);
            tableRow.put("extensions", null);
            tableRow.put("flags", ImmutableSet.of("compound"));
            tableRow.put("gc_grace_seconds", 864000);
            tableRow.put("id", UUID.randomUUID());
            tableRow.put("max_index_interval", 2048);
            tableRow.put("memtable_flush_period_in_ms", 0);
            tableRow.put("min_index_interval", 128);
            tableRow.put("read_repair_chance", 0d);
            tableRow.put("speculative_retry", "99PERCENTILE");
            allTablesRows.add(tableRow);
            Query whenSelectTable = new Query(SELECT_TABLES + " WHERE keyspace_name = '" + keyspace.name + "'  AND table_name = '" + table.name + '\'');
            SuccessResult thenReturnTable = new SuccessResult(Collections.singletonList(tableRow), new LinkedHashMap<>(TABLE_COLUMNS));
            RequestPrime primeTable = new RequestPrime(whenSelectTable, thenReturnTable);
            simulacron.prime(new Prime(primeTable));
            List<LinkedHashMap<String, Object>> tableColumnsRows = new ArrayList<>();
            int position = 0;
            for (Column column : table.partitionKey) {
                LinkedHashMap<String, Object> columnRow = new LinkedHashMap<>();
                columnRow.put("keyspace_name", keyspace.name);
                columnRow.put("table_name", table.name);
                columnRow.put("column_name", column.name);
                columnRow.put("clustering_order", "none");
                columnRow.put("column_name_bytes", column.name.getBytes(StandardCharsets.UTF_8));
                columnRow.put("kind", "partition_key");
                columnRow.put("position", position++);
                columnRow.put("type", column.getTypeAsString());
                tableColumnsRows.add(columnRow);
            }
            position = 0;
            for (Column column : table.clusteringColumns) {
                LinkedHashMap<String, Object> columnRow = new LinkedHashMap<>();
                columnRow.put("keyspace_name", keyspace.name);
                columnRow.put("table_name", table.name);
                columnRow.put("column_name", column.name);
                columnRow.put("clustering_order", "asc");
                columnRow.put("column_name_bytes", column.name.getBytes(StandardCharsets.UTF_8));
                columnRow.put("kind", "clustering");
                columnRow.put("position", position++);
                columnRow.put("type", column.getTypeAsString());
                tableColumnsRows.add(columnRow);
            }
            for (Column column : table.otherColumns) {
                LinkedHashMap<String, Object> columnRow = new LinkedHashMap<>();
                columnRow.put("keyspace_name", keyspace.name);
                columnRow.put("table_name", table.name);
                columnRow.put("column_name", column.name);
                columnRow.put("clustering_order", "none");
                columnRow.put("column_name_bytes", column.name.getBytes(StandardCharsets.UTF_8));
                columnRow.put("kind", "regular");
                columnRow.put("position", -1);
                columnRow.put("type", column.getTypeAsString());
                tableColumnsRows.add(columnRow);
            }
            Query whenSelectTableColumns = new Query(SELECT_COLUMNS + " WHERE keyspace_name = '" + keyspace.name + "'  AND table_name = '" + table.name + '\'');
            SuccessResult thenReturnTableColumns = new SuccessResult(tableColumnsRows, new LinkedHashMap<>(TABLE_COLUMNS));
            RequestPrime primeAllTableColumns = new RequestPrime(whenSelectTableColumns, thenReturnTableColumns);
            simulacron.prime(new Prime(primeAllTableColumns));
            allColumnsRows.addAll(tableColumnsRows);
            // INSERT INTO table
            Query whenInsertIntoTable = new Query(String.format("INSERT INTO %s.%s (%s) VALUES (%s)", asCql(keyspace.name), asCql(table.name), table.allColumns().stream().map(col -> asCql(col.name)).collect(COMMA), table.allColumns().stream().map(col -> ":" + asCql(col.name)).collect(COMMA)), emptyList(), new LinkedHashMap<String, Object>(), table.allColumnTypes());
            simulacron.prime(new Prime(new RequestPrime(whenInsertIntoTable, new SuccessResult(emptyList(), new LinkedHashMap<String, String>()))));
            // UPDATE table
            Query whenUpdateIntoTable = new Query(String.format("UPDATE %s.%s SET %s", asCql(keyspace.name), asCql(table.name), table.allColumns().stream().map(col -> asCql(col.name) + "=:" + asCql(col.name)).collect(COMMA)), emptyList(), new LinkedHashMap<String, Object>(), table.allColumnTypes());
            simulacron.prime(new Prime(new RequestPrime(whenUpdateIntoTable, new SuccessResult(emptyList(), new LinkedHashMap<String, String>()))));
            // SELECT cols from table
            Query whenSelectFromTable = new Query(String.format("SELECT %s FROM %s.%s", table.allColumns().stream().map(col -> asCql(col.name)).collect(COMMA), asCql(keyspace.name), asCql(table.name)));
            simulacron.prime(new Prime(new RequestPrime(whenSelectFromTable, new SuccessResult(table.rows, table.allColumnTypes()))));
            // SELECT from table WHERE token...
            Query whenSelectFromTableWhere = new Query(String.format("SELECT %s FROM %s.%s WHERE token(%s) > ? AND token(%s) <= ?", table.allColumns().stream().map(col -> asCql(col.name)).collect(COMMA), asCql(keyspace.name), asCql(table.name), table.partitionKey.stream().map(col -> asCql(col.name)).collect(COMMA), table.partitionKey.stream().map(col -> asCql(col.name)).collect(COMMA)));
            simulacron.prime(new Prime(new RequestPrime(whenSelectFromTableWhere, new SuccessResult(table.rows, table.allColumnTypes()))));
            whenSelectFromTableWhere = new Query(String.format("SELECT %s FROM %s.%s WHERE token(%s) > :start AND token(%s) <= :end", table.allColumns().stream().map(col -> asCql(col.name)).collect(COMMA), asCql(keyspace.name), asCql(table.name), table.partitionKey.stream().map(col -> asCql(col.name)).collect(COMMA), table.partitionKey.stream().map(col -> asCql(col.name)).collect(COMMA)));
            simulacron.prime(new Prime(new RequestPrime(whenSelectFromTableWhere, new SuccessResult(table.rows, table.allColumnTypes()))));
        }
    }
    Query whenSelectAllKeyspaces = new Query(SELECT_KEYSPACES);
    SuccessResult thenReturnAllKeyspaces = new SuccessResult(allKeyspacesRows, new LinkedHashMap<>(KEYSPACE_COLUMNS));
    RequestPrime primeAllKeyspaces = new RequestPrime(whenSelectAllKeyspaces, thenReturnAllKeyspaces);
    simulacron.prime(new Prime(primeAllKeyspaces));
    Query whenSelectAllTables = new Query(SELECT_TABLES);
    SuccessResult thenReturnAllTables = new SuccessResult(allTablesRows, new LinkedHashMap<>(TABLE_COLUMNS));
    RequestPrime primeAllTables = new RequestPrime(whenSelectAllTables, thenReturnAllTables);
    simulacron.prime(new Prime(primeAllTables));
    Query whenSelectAllColumns = new Query(SELECT_COLUMNS);
    SuccessResult thenReturnAllColumns = new SuccessResult(allColumnsRows, new LinkedHashMap<>(COLUMN_COLUMNS));
    RequestPrime primeAllColumns = new RequestPrime(whenSelectAllColumns, thenReturnAllColumns);
    simulacron.prime(new Prime(primeAllColumns));
}