Examples with MerkleTrees org.apache.cassandra.utils.MerkleTrees used on opensource projects

Example 6 with MerkleTrees

use of org.apache.cassandra.utils.MerkleTrees in project cassandra by apache.

the class LocalSyncTaskTest method testDifference.

@Test
public void testDifference() throws Throwable {
    Range<Token> range = new Range<>(partitioner.getMinimumToken(), partitioner.getRandomToken());
    UUID parentRepairSession = UUID.randomUUID();
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
    ActiveRepairService.instance.registerParentRepairSession(parentRepairSession, FBUtilities.getBroadcastAddressAndPort(), Arrays.asList(cfs), Arrays.asList(range), false, ActiveRepairService.UNREPAIRED_SSTABLE, false, PreviewKind.NONE);
    RepairJobDesc desc = new RepairJobDesc(parentRepairSession, UUID.randomUUID(), KEYSPACE1, "Standard1", Arrays.asList(range));
    MerkleTrees tree1 = createInitialTree(desc);
    MerkleTrees tree2 = createInitialTree(desc);
    // change a range in one of the trees
    Token token = partitioner.midpoint(range.left, range.right);
    tree1.invalidate(token);
    MerkleTree.TreeRange changed = tree1.get(token);
    changed.hash("non-empty hash!".getBytes());
    Set<Range<Token>> interesting = new HashSet<>();
    interesting.add(changed);
    // difference the trees
    // note: we reuse the same endpoint which is bogus in theory but fine here
    TreeResponse r1 = new TreeResponse(local, tree1);
    TreeResponse r2 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.2"), tree2);
    LocalSyncTask task = new LocalSyncTask(desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
    NettyStreamingConnectionFactory.MAX_CONNECT_ATTEMPTS = 1;
    try {
        task.run();
    } finally {
        NettyStreamingConnectionFactory.MAX_CONNECT_ATTEMPTS = 3;
    }
    // ensure that the changed range was recorded
    assertEquals("Wrong differing ranges", interesting.size(), task.stat.numberOfDifferences);
}

Example 7 with MerkleTrees

use of org.apache.cassandra.utils.MerkleTrees in project cassandra by apache.

the class RepairMessageSerializationsTest method validationCompleteMessage_WithMerkleTree.

@Test
public void validationCompleteMessage_WithMerkleTree() throws IOException {
    MerkleTrees trees = new MerkleTrees(Murmur3Partitioner.instance);
    trees.addMerkleTree(256, new Range<>(new LongToken(1000), new LongToken(1001)));
    ValidationResponse deserialized = validationCompleteMessage(trees);
    // a simple check to make sure we got some merkle trees back.
    Assert.assertEquals(trees.size(), deserialized.trees.size());
}

Example 8 with MerkleTrees

use of org.apache.cassandra.utils.MerkleTrees in project cassandra by apache.

the class DifferenceHolderTest method testFromEmptyMerkleTrees.

@Test
public void testFromEmptyMerkleTrees() throws UnknownHostException {
    InetAddressAndPort a1 = InetAddressAndPort.getByName("127.0.0.1");
    InetAddressAndPort a2 = InetAddressAndPort.getByName("127.0.0.2");
    MerkleTrees mts1 = new MerkleTrees(Murmur3Partitioner.instance);
    MerkleTrees mts2 = new MerkleTrees(Murmur3Partitioner.instance);
    mts1.init();
    mts2.init();
    TreeResponse tr1 = new TreeResponse(a1, mts1);
    TreeResponse tr2 = new TreeResponse(a2, mts2);
    DifferenceHolder dh = new DifferenceHolder(Lists.newArrayList(tr1, tr2));
    assertTrue(dh.get(a1).get(a2).isEmpty());
}

Example 9 with MerkleTrees

use of org.apache.cassandra.utils.MerkleTrees in project cassandra by apache.

the class SerializationsTest method testValidationCompleteWrite.

private void testValidationCompleteWrite() throws IOException {
    IPartitioner p = RandomPartitioner.instance;
    MerkleTrees mts = new MerkleTrees(p);
    // empty validation
    mts.addMerkleTree((int) Math.pow(2, 15), FULL_RANGE);
    Validator v0 = new Validator(DESC, FBUtilities.getBroadcastAddressAndPort(), -1, PreviewKind.NONE);
    ValidationResponse c0 = new ValidationResponse(DESC, mts);
    // validation with a tree
    mts = new MerkleTrees(p);
    mts.addMerkleTree(Integer.MAX_VALUE, FULL_RANGE);
    for (int i = 0; i < 10; i++) mts.split(p.getRandomToken());
    Validator v1 = new Validator(DESC, FBUtilities.getBroadcastAddressAndPort(), -1, PreviewKind.NONE);
    ValidationResponse c1 = new ValidationResponse(DESC, mts);
    // validation failed
    ValidationResponse c3 = new ValidationResponse(DESC);
    testRepairMessageWrite("service.ValidationComplete.bin", ValidationResponse.serializer, c0, c1, c3);
}

Example 10 with MerkleTrees

use of org.apache.cassandra.utils.MerkleTrees in project cassandra by apache.

the class ValidationManager method createMerkleTrees.

private static MerkleTrees createMerkleTrees(ValidationPartitionIterator validationIterator, Collection<Range<Token>> ranges, ColumnFamilyStore cfs) {
    MerkleTrees trees = new MerkleTrees(cfs.getPartitioner());
    long allPartitions = validationIterator.estimatedPartitions();
    Map<Range<Token>, Long> rangePartitionCounts = validationIterator.getRangePartitionCounts();
    // The repair coordinator must hold RF trees in memory at once, so a given validation compaction can only
    // use 1 / RF of the allowed space.
    long availableBytes = (DatabaseDescriptor.getRepairSessionSpaceInMiB() * 1048576) / cfs.keyspace.getReplicationStrategy().getReplicationFactor().allReplicas;
    for (Range<Token> range : ranges) {
        long numPartitions = rangePartitionCounts.get(range);
        double rangeOwningRatio = allPartitions > 0 ? (double) numPartitions / allPartitions : 0;
        // determine max tree depth proportional to range size to avoid blowing up memory with multiple tress,
        // capping at a depth that does not exceed our memory budget (CASSANDRA-11390, CASSANDRA-14096)
        int rangeAvailableBytes = Math.max(1, (int) (rangeOwningRatio * availableBytes));
        // Try to estimate max tree depth that fits the space budget assuming hashes of 256 bits = 32 bytes
        // note that estimatedMaxDepthForBytes cannot return a number lower than 1
        int estimatedMaxDepth = MerkleTree.estimatedMaxDepthForBytes(cfs.getPartitioner(), rangeAvailableBytes, 32);
        int maxDepth = rangeOwningRatio > 0 ? Math.min(estimatedMaxDepth, DatabaseDescriptor.getRepairSessionMaxTreeDepth()) : 0;
        // determine tree depth from number of partitions, capping at max tree depth (CASSANDRA-5263)
        int depth = numPartitions > 0 ? (int) Math.min(Math.ceil(Math.log(numPartitions) / Math.log(2)), maxDepth) : 0;
        trees.addMerkleTree((int) Math.pow(2, depth), range);
    }
    if (logger.isDebugEnabled()) {
        // MT serialize may take time
        logger.debug("Created {} merkle trees with merkle trees size {}, {} partitions, {} bytes", trees.ranges().size(), trees.size(), allPartitions, MerkleTrees.serializer.serializedSize(trees, 0));
    }
    return trees;
}