Examples with ClusteringComparator org.apache.cassandra.db.ClusteringComparator used on opensource projects

Example 11 with ClusteringComparator

use of org.apache.cassandra.db.ClusteringComparator in project cassandra by apache.

the class CompressedSequentialWriterTest method testWrite.

private void testWrite(File f, int bytesToTest, boolean useMemmap) throws IOException {
    final String filename = f.getAbsolutePath();
    MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Collections.singletonList(BytesType.instance)));
    byte[] dataPre = new byte[bytesToTest];
    byte[] rawPost = new byte[bytesToTest];
    try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", null, SequentialWriterOption.DEFAULT, compressionParameters, sstableMetadataCollector)) {
        Random r = new Random(42);
        // Test both write with byte[] and ByteBuffer
        r.nextBytes(dataPre);
        r.nextBytes(rawPost);
        ByteBuffer dataPost = makeBB(bytesToTest);
        dataPost.put(rawPost);
        dataPost.flip();
        writer.write(dataPre);
        DataPosition mark = writer.mark();
        // Write enough garbage to transition chunk
        for (int i = 0; i < CompressionParams.DEFAULT_CHUNK_LENGTH; i++) {
            writer.write((byte) i);
        }
        writer.resetAndTruncate(mark);
        writer.write(dataPost);
        writer.finish();
    }
    assert f.exists();
    try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
        FileHandle fh = builder.complete();
        RandomAccessReader reader = fh.createReader()) {
        assertEquals(dataPre.length + rawPost.length, reader.length());
        byte[] result = new byte[(int) reader.length()];
        reader.readFully(result);
        assert (reader.isEOF());
        reader.close();
        byte[] fullInput = new byte[bytesToTest * 2];
        System.arraycopy(dataPre, 0, fullInput, 0, dataPre.length);
        System.arraycopy(rawPost, 0, fullInput, bytesToTest, rawPost.length);
        assert Arrays.equals(result, fullInput);
    } finally {
        if (f.exists())
            f.delete();
        File metadata = new File(f + ".metadata");
        if (metadata.exists())
            metadata.delete();
    }
}

Example 12 with ClusteringComparator

use of org.apache.cassandra.db.ClusteringComparator in project cassandra by apache.

the class MmappedRegionsTest method testMapForCompressionMetadata.

@Test
public void testMapForCompressionMetadata() throws Exception {
    int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
    MmappedRegions.MAX_SEGMENT_SIZE = 1024;
    ByteBuffer buffer = allocateBuffer(128 * 1024);
    File f = File.createTempFile("testMapForCompressionMetadata", "1");
    f.deleteOnExit();
    File cf = File.createTempFile(f.getName() + ".metadata", "1");
    cf.deleteOnExit();
    MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
    try (SequentialWriter writer = new CompressedSequentialWriter(f, cf.getAbsolutePath(), null, SequentialWriterOption.DEFAULT, CompressionParams.snappy(), sstableMetadataCollector)) {
        writer.write(buffer);
        writer.finish();
    }
    CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), true);
    try (ChannelProxy channel = new ChannelProxy(f);
        MmappedRegions regions = MmappedRegions.map(channel, metadata)) {
        assertFalse(regions.isEmpty());
        int i = 0;
        while (i < buffer.capacity()) {
            CompressionMetadata.Chunk chunk = metadata.chunkFor(i);
            MmappedRegions.Region region = regions.floor(chunk.offset);
            assertNotNull(region);
            ByteBuffer compressedChunk = region.buffer.duplicate();
            assertNotNull(compressedChunk);
            assertEquals(chunk.length + 4, compressedChunk.capacity());
            assertEquals(chunk.offset, region.offset());
            assertEquals(chunk.offset + chunk.length + 4, region.end());
            i += metadata.chunkLength();
        }
    } finally {
        MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
        metadata.close();
    }
}

Example 13 with ClusteringComparator

use of org.apache.cassandra.db.ClusteringComparator in project cassandra by apache.

the class CompressedInputStreamTest method testCompressedReadWith.

/**
     * @param valuesToCheck array of longs of range(0-999)
     * @throws Exception
     */
private void testCompressedReadWith(long[] valuesToCheck, boolean testTruncate, boolean testException, double minCompressRatio) throws Exception {
    assert valuesToCheck != null && valuesToCheck.length > 0;
    // write compressed data file of longs
    File parentDir = new File(System.getProperty("java.io.tmpdir"));
    Descriptor desc = new Descriptor(parentDir, "ks", "cf", 1);
    File tmp = new File(desc.filenameFor(Component.DATA));
    MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
    CompressionParams param = CompressionParams.snappy(32, minCompressRatio);
    Map<Long, Long> index = new HashMap<Long, Long>();
    try (CompressedSequentialWriter writer = new CompressedSequentialWriter(tmp, desc.filenameFor(Component.COMPRESSION_INFO), null, SequentialWriterOption.DEFAULT, param, collector)) {
        for (long l = 0L; l < 1000; l++) {
            index.put(l, writer.position());
            writer.writeLong(l);
        }
        writer.finish();
    }
    CompressionMetadata comp = CompressionMetadata.create(tmp.getAbsolutePath());
    List<Pair<Long, Long>> sections = new ArrayList<>();
    for (long l : valuesToCheck) {
        long position = index.get(l);
        sections.add(Pair.create(position, position + 8));
    }
    CompressionMetadata.Chunk[] chunks = comp.getChunksForSections(sections);
    long totalSize = comp.getTotalSizeForSections(sections);
    long expectedSize = 0;
    for (CompressionMetadata.Chunk c : chunks) expectedSize += c.length + 4;
    assertEquals(expectedSize, totalSize);
    // buffer up only relevant parts of file
    int size = 0;
    for (CompressionMetadata.Chunk c : chunks) // 4bytes CRC
    size += (c.length + 4);
    byte[] toRead = new byte[size];
    try (RandomAccessFile f = new RandomAccessFile(tmp, "r")) {
        int pos = 0;
        for (CompressionMetadata.Chunk c : chunks) {
            f.seek(c.offset);
            pos += f.read(toRead, pos, c.length + 4);
        }
    }
    if (testTruncate) {
        byte[] actuallyRead = new byte[50];
        System.arraycopy(toRead, 0, actuallyRead, 0, 50);
        toRead = actuallyRead;
    }
    // read buffer using CompressedInputStream
    CompressionInfo info = new CompressionInfo(chunks, param);
    if (testException) {
        testException(sections, info);
        return;
    }
    CompressedInputStream input = new CompressedInputStream(new ByteArrayInputStream(toRead), info, ChecksumType.CRC32, () -> 1.0);
    try (DataInputStream in = new DataInputStream(input)) {
        for (int i = 0; i < sections.size(); i++) {
            input.position(sections.get(i).left);
            long readValue = in.readLong();
            assertEquals("expected " + valuesToCheck[i] + " but was " + readValue, valuesToCheck[i], readValue);
        }
    }
}