Examples with Range org.apache.cassandra.dht.Range used on opensource projects

Example 96 with Range

use of org.apache.cassandra.dht.Range in project cassandra by apache.

the class RepairOption method parse.

/**
     * Construct RepairOptions object from given map of Strings.
     * <p>
     * Available options are:
     *
     * <table>
     *     <caption>Repair Options</caption>
     *     <thead>
     *         <tr>
     *             <th>key</th>
     *             <th>value</th>
     *             <th>default (when key not given)</th>
     *         </tr>
     *     </thead>
     *     <tbody>
     *         <tr>
     *             <td>parallelism</td>
     *             <td>"sequential", "parallel" or "dc_parallel"</td>
     *             <td>"sequential"</td>
     *         </tr>
     *         <tr>
     *             <td>primaryRange</td>
     *             <td>"true" if perform repair only on primary range.</td>
     *             <td>false</td>
     *         </tr>
     *         <tr>
     *             <td>incremental</td>
     *             <td>"true" if perform incremental repair.</td>
     *             <td>false</td>
     *         </tr>
     *         <tr>
     *             <td>trace</td>
     *             <td>"true" if repair is traced.</td>
     *             <td>false</td>
     *         </tr>
     *         <tr>
     *             <td>jobThreads</td>
     *             <td>Number of threads to use to run repair job.</td>
     *             <td>1</td>
     *         </tr>
     *         <tr>
     *             <td>ranges</td>
     *             <td>Ranges to repair. A range is expressed as &lt;start token&gt;:&lt;end token&gt;
     *             and multiple ranges can be given as comma separated ranges(e.g. aaa:bbb,ccc:ddd).</td>
     *             <td></td>
     *         </tr>
     *         <tr>
     *             <td>columnFamilies</td>
     *             <td>Specify names of ColumnFamilies to repair.
     *             Multiple ColumnFamilies can be given as comma separated values(e.g. cf1,cf2,cf3).</td>
     *             <td></td>
     *         </tr>
     *         <tr>
     *             <td>dataCenters</td>
     *             <td>Specify names of data centers who participate in this repair.
     *             Multiple data centers can be given as comma separated values(e.g. dc1,dc2,dc3).</td>
     *             <td></td>
     *         </tr>
     *         <tr>
     *             <td>hosts</td>
     *             <td>Specify names of hosts who participate in this repair.
     *             Multiple hosts can be given as comma separated values(e.g. cass1,cass2).</td>
     *             <td></td>
     *         </tr>
     *         <tr>
     *             <td>pullRepair</td>
     *             <td>"true" if the repair should only stream data one way from a remote host to this host.
     *             This is only allowed if exactly 2 hosts are specified along with a token range that they share.</td>
     *             <td>false</td>
     *         </tr>
     *     </tbody>
     * </table>
     *
     * @param options options to parse
     * @param partitioner partitioner is used to construct token ranges
     * @return RepairOptions object
     */
public static RepairOption parse(Map<String, String> options, IPartitioner partitioner) {
    // if no parallel option is given, then this will be "sequential" by default.
    RepairParallelism parallelism = RepairParallelism.fromName(options.get(PARALLELISM_KEY));
    boolean primaryRange = Boolean.parseBoolean(options.get(PRIMARY_RANGE_KEY));
    boolean incremental = Boolean.parseBoolean(options.get(INCREMENTAL_KEY));
    boolean trace = Boolean.parseBoolean(options.get(TRACE_KEY));
    boolean pullRepair = Boolean.parseBoolean(options.get(PULL_REPAIR_KEY));
    int jobThreads = 1;
    if (options.containsKey(JOB_THREADS_KEY)) {
        try {
            jobThreads = Integer.parseInt(options.get(JOB_THREADS_KEY));
        } catch (NumberFormatException ignore) {
        }
    }
    // ranges
    String rangesStr = options.get(RANGES_KEY);
    Set<Range<Token>> ranges = new HashSet<>();
    if (rangesStr != null) {
        if (incremental)
            logger.warn("Incremental repair can't be requested with subrange repair " + "because each subrange repair would generate an anti-compacted table. " + "The repair will occur but without anti-compaction.");
        StringTokenizer tokenizer = new StringTokenizer(rangesStr, ",");
        while (tokenizer.hasMoreTokens()) {
            String[] rangeStr = tokenizer.nextToken().split(":", 2);
            if (rangeStr.length < 2) {
                continue;
            }
            Token parsedBeginToken = partitioner.getTokenFactory().fromString(rangeStr[0].trim());
            Token parsedEndToken = partitioner.getTokenFactory().fromString(rangeStr[1].trim());
            ranges.add(new Range<>(parsedBeginToken, parsedEndToken));
        }
    }
    RepairOption option = new RepairOption(parallelism, primaryRange, incremental, trace, jobThreads, ranges, !ranges.isEmpty(), pullRepair);
    // data centers
    String dataCentersStr = options.get(DATACENTERS_KEY);
    Collection<String> dataCenters = new HashSet<>();
    if (dataCentersStr != null) {
        StringTokenizer tokenizer = new StringTokenizer(dataCentersStr, ",");
        while (tokenizer.hasMoreTokens()) {
            dataCenters.add(tokenizer.nextToken().trim());
        }
        option.getDataCenters().addAll(dataCenters);
    }
    // hosts
    String hostsStr = options.get(HOSTS_KEY);
    Collection<String> hosts = new HashSet<>();
    if (hostsStr != null) {
        StringTokenizer tokenizer = new StringTokenizer(hostsStr, ",");
        while (tokenizer.hasMoreTokens()) {
            hosts.add(tokenizer.nextToken().trim());
        }
        option.getHosts().addAll(hosts);
    }
    // columnfamilies
    String cfStr = options.get(COLUMNFAMILIES_KEY);
    if (cfStr != null) {
        Collection<String> columnFamilies = new HashSet<>();
        StringTokenizer tokenizer = new StringTokenizer(cfStr, ",");
        while (tokenizer.hasMoreTokens()) {
            columnFamilies.add(tokenizer.nextToken().trim());
        }
        option.getColumnFamilies().addAll(columnFamilies);
    }
    // validate options
    if (jobThreads > MAX_JOB_THREADS) {
        throw new IllegalArgumentException("Too many job threads. Max is " + MAX_JOB_THREADS);
    }
    if (!dataCenters.isEmpty() && !hosts.isEmpty()) {
        throw new IllegalArgumentException("Cannot combine -dc and -hosts options.");
    }
    if (primaryRange && ((!dataCenters.isEmpty() && !option.isInLocalDCOnly()) || !hosts.isEmpty())) {
        throw new IllegalArgumentException("You need to run primary range repair on all nodes in the cluster.");
    }
    if (pullRepair) {
        if (hosts.size() != 2) {
            throw new IllegalArgumentException("Pull repair can only be performed between two hosts. Please specify two hosts, one of which must be this host.");
        } else if (ranges.isEmpty()) {
            throw new IllegalArgumentException("Token ranges must be specified when performing pull repair. Please specify at least one token range which both hosts have in common.");
        }
    }
    if (option.isIncremental() && !option.isGlobal()) {
        throw new IllegalArgumentException("Incremental repairs cannot be run against a subset of tokens or ranges");
    }
    return option;
}

Example 97 with Range

use of org.apache.cassandra.dht.Range in project cassandra by apache.

the class RepairRunnable method consistentRepair.

private void consistentRepair(UUID parentSession, long repairedAt, long startTime, TraceState traceState, Set<InetAddress> allNeighbors, List<Pair<Set<InetAddress>, ? extends Collection<Range<Token>>>> commonRanges, String... cfnames) {
    // the local node also needs to be included in the set of
    // participants, since coordinator sessions aren't persisted
    Set<InetAddress> allParticipants = new HashSet<>(allNeighbors);
    allParticipants.add(FBUtilities.getBroadcastAddress());
    CoordinatorSession coordinatorSession = ActiveRepairService.instance.consistent.coordinated.registerSession(parentSession, allParticipants);
    ListeningExecutorService executor = createExecutor();
    AtomicBoolean hasFailure = new AtomicBoolean(false);
    ListenableFuture repairResult = coordinatorSession.execute(executor, () -> submitRepairSessions(parentSession, repairedAt, true, executor, commonRanges, cfnames), hasFailure);
    Collection<Range<Token>> ranges = new HashSet<>();
    for (Collection<Range<Token>> range : Iterables.transform(commonRanges, cr -> cr.right)) {
        ranges.addAll(range);
    }
    Futures.addCallback(repairResult, new RepairCompleteCallback(parentSession, ranges, startTime, traceState, hasFailure, executor));
}

Example 98 with Range

use of org.apache.cassandra.dht.Range in project cassandra by apache.

the class RepairRunnable method normalRepair.

private void normalRepair(UUID parentSession, long startTime, TraceState traceState, Set<InetAddress> allNeighbors, List<Pair<Set<InetAddress>, ? extends Collection<Range<Token>>>> commonRanges, String... cfnames) {
    // Set up RepairJob executor for this repair command.
    ListeningExecutorService executor = createExecutor();
    // Setting the repairedAt time to UNREPAIRED_SSTABLE causes the repairedAt times to be preserved across streamed sstables
    final ListenableFuture<List<RepairSessionResult>> allSessions = submitRepairSessions(parentSession, ActiveRepairService.UNREPAIRED_SSTABLE, false, executor, commonRanges, cfnames);
    // After all repair sessions completes(successful or not),
    // run anticompaction if necessary and send finish notice back to client
    final Collection<Range<Token>> successfulRanges = new ArrayList<>();
    final AtomicBoolean hasFailure = new AtomicBoolean();
    ListenableFuture repairResult = Futures.transform(allSessions, new AsyncFunction<List<RepairSessionResult>, Object>() {

        @SuppressWarnings("unchecked")
        public ListenableFuture apply(List<RepairSessionResult> results) {
            // filter out null(=failed) results and get successful ranges
            for (RepairSessionResult sessionResult : results) {
                if (sessionResult != null) {
                    successfulRanges.addAll(sessionResult.ranges);
                } else {
                    hasFailure.compareAndSet(false, true);
                }
            }
            return Futures.immediateFuture(null);
        }
    });
    Futures.addCallback(repairResult, new RepairCompleteCallback(parentSession, successfulRanges, startTime, traceState, hasFailure, executor));
}

Example 99 with Range

use of org.apache.cassandra.dht.Range in project cassandra by apache.

the class LocalSessions method deserializeRange.

private static Range<Token> deserializeRange(ByteBuffer bb) {
    try (DataInputBuffer in = new DataInputBuffer(bb, false)) {
        IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
        Token left = Token.serializer.deserialize(in, partitioner, 0);
        Token right = Token.serializer.deserialize(in, partitioner, 0);
        return new Range<>(left, right);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 100 with Range

use of org.apache.cassandra.dht.Range in project cassandra by apache.

the class LongStreamingTest method testCompressedStream.

@Test
public void testCompressedStream() throws InvalidRequestException, IOException, ExecutionException, InterruptedException {
    String KS = "cql_keyspace";
    String TABLE = "table1";
    File tempdir = Files.createTempDir();
    File dataDir = new File(tempdir.getAbsolutePath() + File.separator + KS + File.separator + TABLE);
    assert dataDir.mkdirs();
    String schema = "CREATE TABLE cql_keyspace.table1 (" + "  k int PRIMARY KEY," + "  v1 text," + "  v2 int" + // with compression = {};";
    ");";
    String insert = "INSERT INTO cql_keyspace.table1 (k, v1, v2) VALUES (?, ?, ?)";
    CQLSSTableWriter writer = CQLSSTableWriter.builder().sorted().inDirectory(dataDir).forTable(schema).using(insert).build();
    long start = System.nanoTime();
    for (int i = 0; i < 10_000_000; i++) writer.addRow(i, "test1", 24);
    writer.close();
    System.err.println(String.format("Writer finished after %d seconds....", TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start)));
    File[] dataFiles = dataDir.listFiles((dir, name) -> name.endsWith("-Data.db"));
    long dataSize = 0l;
    for (File file : dataFiles) {
        System.err.println("File : " + file.getAbsolutePath());
        dataSize += file.length();
    }
    SSTableLoader loader = new SSTableLoader(dataDir, new SSTableLoader.Client() {

        private String ks;

        public void init(String keyspace) {
            for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace")) addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
            this.ks = keyspace;
        }

        public TableMetadataRef getTableMetadata(String cfName) {
            return Schema.instance.getTableMetadataRef(ks, cfName);
        }
    }, new OutputHandler.SystemOutput(false, false));
    start = System.nanoTime();
    loader.stream().get();
    long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.err.println(String.format("Finished Streaming in %.2f seconds: %.2f Mb/sec", millis / 1000d, (dataSize / (1 << 20) / (millis / 1000d)) * 8));
    //Stream again
    loader = new SSTableLoader(dataDir, new SSTableLoader.Client() {

        private String ks;

        public void init(String keyspace) {
            for (Range<Token> range : StorageService.instance.getLocalRanges("cql_keyspace")) addRangeForEndpoint(range, FBUtilities.getBroadcastAddress());
            this.ks = keyspace;
        }

        public TableMetadataRef getTableMetadata(String cfName) {
            return Schema.instance.getTableMetadataRef(ks, cfName);
        }
    }, new OutputHandler.SystemOutput(false, false));
    start = System.nanoTime();
    loader.stream().get();
    millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.err.println(String.format("Finished Streaming in %.2f seconds: %.2f Mb/sec", millis / 1000d, (dataSize / (1 << 20) / (millis / 1000d)) * 8));
    //Compact them both
    start = System.nanoTime();
    Keyspace.open(KS).getColumnFamilyStore(TABLE).forceMajorCompaction();
    millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.err.println(String.format("Finished Compacting in %.2f seconds: %.2f Mb/sec", millis / 1000d, (dataSize * 2 / (1 << 20) / (millis / 1000d)) * 8));
    UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM cql_keyspace.table1 limit 100;");
    assertEquals(100, rs.size());
}