Examples with CompressionParams org.apache.cassandra.schema.CompressionParams used on opensource projects

Example 1 with CompressionParams

use of org.apache.cassandra.schema.CompressionParams in project cassandra by apache.

the class VerifyTest method defineSchema.

@BeforeClass
public static void defineSchema() throws ConfigurationException {
    CompressionParams compressionParameters = CompressionParams.snappy(32768);
    loadSchema();
    createKeyspace(KEYSPACE, KeyspaceParams.simple(1), standardCFMD(KEYSPACE, CF).compression(compressionParameters), standardCFMD(KEYSPACE, CF2).compression(compressionParameters), standardCFMD(KEYSPACE, CF3), standardCFMD(KEYSPACE, CF4), standardCFMD(KEYSPACE, CORRUPT_CF), standardCFMD(KEYSPACE, CORRUPT_CF2), counterCFMD(KEYSPACE, COUNTER_CF).compression(compressionParameters), counterCFMD(KEYSPACE, COUNTER_CF2).compression(compressionParameters), counterCFMD(KEYSPACE, COUNTER_CF3), counterCFMD(KEYSPACE, COUNTER_CF4), counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF), counterCFMD(KEYSPACE, CORRUPTCOUNTER_CF2), standardCFMD(KEYSPACE, CF_UUID, 0, UUIDType.instance), standardCFMD(KEYSPACE, BF_ALWAYS_PRESENT).bloomFilterFpChance(1.0));
}

Example 2 with CompressionParams

use of org.apache.cassandra.schema.CompressionParams in project cassandra by apache.

the class LongStreamingTest method testStream.

private void testStream(boolean useSstableCompression) throws InvalidRequestException, IOException, ExecutionException, InterruptedException {
    String KS = useSstableCompression ? "sstable_compression_ks" : "stream_compression_ks";
    String TABLE = "table1";
    File tempdir = new File(Files.createTempDir());
    File dataDir = new File(tempdir.absolutePath() + File.pathSeparator() + KS + File.pathSeparator() + TABLE);
    assert dataDir.tryCreateDirectories();
    String schema = "CREATE TABLE " + KS + '.' + TABLE + "  (" + "  k int PRIMARY KEY," + "  v1 text," + "  v2 int" + ") with compression = " + (useSstableCompression ? "{'class': 'LZ4Compressor'};" : "{};");
    String insert = "INSERT INTO " + KS + '.' + TABLE + " (k, v1, v2) VALUES (?, ?, ?)";
    CQLSSTableWriter writer = CQLSSTableWriter.builder().sorted().inDirectory(dataDir).forTable(schema).using(insert).build();
    CompressionParams compressionParams = Keyspace.open(KS).getColumnFamilyStore(TABLE).metadata().params.compression;
    Assert.assertEquals(useSstableCompression, compressionParams.isEnabled());
    long start = nanoTime();
    for (int i = 0; i < 10_000_000; i++) writer.addRow(i, "test1", 24);
    writer.close();
    System.err.println(String.format("Writer finished after %d seconds....", TimeUnit.NANOSECONDS.toSeconds(nanoTime() - start)));
    File[] dataFiles = dataDir.tryList((dir, name) -> name.endsWith("-Data.db"));
    long dataSizeInBytes = 0l;
    for (File file : dataFiles) {
        System.err.println("File : " + file.absolutePath());
        dataSizeInBytes += file.length();
    }
    SSTableLoader loader = new SSTableLoader(dataDir, new SSTableLoader.Client() {

        private String ks;

        public void init(String keyspace) {
            for (Replica range : StorageService.instance.getLocalReplicas(KS)) addRangeForEndpoint(range.range(), FBUtilities.getBroadcastAddressAndPort());
            this.ks = keyspace;
        }

        public TableMetadataRef getTableMetadata(String cfName) {
            return Schema.instance.getTableMetadataRef(ks, cfName);
        }
    }, new OutputHandler.SystemOutput(false, false));
    start = nanoTime();
    loader.stream().get();
    long millis = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
    System.err.println(String.format("Finished Streaming in %.2f seconds: %.2f MiBsec", millis / 1000d, (dataSizeInBytes / (1 << 20) / (millis / 1000d)) * 8));
    // Stream again
    loader = new SSTableLoader(dataDir, new SSTableLoader.Client() {

        private String ks;

        public void init(String keyspace) {
            for (Replica range : StorageService.instance.getLocalReplicas(KS)) addRangeForEndpoint(range.range(), FBUtilities.getBroadcastAddressAndPort());
            this.ks = keyspace;
        }

        public TableMetadataRef getTableMetadata(String cfName) {
            return Schema.instance.getTableMetadataRef(ks, cfName);
        }
    }, new OutputHandler.SystemOutput(false, false));
    start = nanoTime();
    loader.stream().get();
    millis = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
    System.err.println(String.format("Finished Streaming in %.2f seconds: %.2f MiBsec", millis / 1000d, (dataSizeInBytes / (1 << 20) / (millis / 1000d)) * 8));
    // Compact them both
    start = nanoTime();
    Keyspace.open(KS).getColumnFamilyStore(TABLE).forceMajorCompaction();
    millis = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
    System.err.println(String.format("Finished Compacting in %.2f seconds: %.2f MiBsec", millis / 1000d, (dataSizeInBytes * 2 / (1 << 20) / (millis / 1000d)) * 8));
    UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM " + KS + '.' + TABLE + " limit 100;");
    assertEquals(100, rs.size());
}

Example 3 with CompressionParams

use of org.apache.cassandra.schema.CompressionParams in project cassandra by apache.

the class BigTableWriter method compressionFor.

/**
 * Given an OpType, determine the correct Compression Parameters
 * @param opType
 * @return {@link org.apache.cassandra.schema.CompressionParams}
 */
private CompressionParams compressionFor(final OperationType opType) {
    CompressionParams compressionParams = metadata.getLocal().params.compression;
    final ICompressor compressor = compressionParams.getSstableCompressor();
    if (null != compressor && opType == OperationType.FLUSH) {
        // compression algorithm indicates we should. See CASSANDRA-15379 for more details.
        switch(DatabaseDescriptor.getFlushCompression()) {
            // We can bypass the CompressedSequentialWriter in this case entirely.
            case none:
                compressionParams = CompressionParams.NOOP;
                break;
            case fast:
                if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION)) {
                    // The default compressor is generally fast (LZ4 with 16KiB block size)
                    compressionParams = CompressionParams.DEFAULT;
                    break;
                }
            case table:
            default:
        }
    }
    return compressionParams;
}

Example 4 with CompressionParams

use of org.apache.cassandra.schema.CompressionParams in project cassandra by apache.

the class CompressedSequentialWriterTest method testUncompressedChunks.

private void testUncompressedChunks(int size, double ratio, int extra) throws IOException {
    // System.out.format("size %d ratio %f extra %d\n", size, ratio, extra);
    ByteBuffer b = ByteBuffer.allocate(size);
    ByteBufferUtil.writeZeroes(b, size);
    b.flip();
    File f = FileUtils.createTempFile("testUncompressedChunks", "1");
    String filename = f.path();
    MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Collections.singletonList(BytesType.instance)));
    compressionParameters = new CompressionParams(MockCompressor.class.getTypeName(), MockCompressor.paramsFor(ratio, extra), DEFAULT_CHUNK_LENGTH, ratio);
    try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, f.path() + ".metadata", null, SequentialWriterOption.DEFAULT, compressionParameters, sstableMetadataCollector)) {
        writer.write(b);
        writer.finish();
        b.flip();
    }
    assert f.exists();
    try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true));
        FileHandle fh = builder.complete();
        RandomAccessReader reader = fh.createReader()) {
        assertEquals(size, reader.length());
        byte[] result = new byte[(int) reader.length()];
        reader.readFully(result);
        assert (reader.isEOF());
        assert Arrays.equals(b.array(), result);
    } finally {
        if (f.exists())
            f.tryDelete();
        File metadata = new File(f + ".metadata");
        if (metadata.exists())
            metadata.tryDelete();
    }
}

Example 5 with CompressionParams

use of org.apache.cassandra.schema.CompressionParams in project cassandra by apache.

the class CompressedInputStreamTest method testCompressedReadWith.

/**
 * @param valuesToCheck array of longs of range(0-999)
 * @throws Exception
 */
private void testCompressedReadWith(long[] valuesToCheck, boolean testTruncate, boolean testException, double minCompressRatio) throws Exception {
    assert valuesToCheck != null && valuesToCheck.length > 0;
    // write compressed data file of longs
    File parentDir = new File(tempFolder.newFolder());
    Descriptor desc = new Descriptor(parentDir, "ks", "cf", 1);
    File tmp = new File(desc.filenameFor(Component.DATA));
    MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
    CompressionParams param = CompressionParams.snappy(32, minCompressRatio);
    Map<Long, Long> index = new HashMap<Long, Long>();
    try (CompressedSequentialWriter writer = new CompressedSequentialWriter(tmp, desc.filenameFor(Component.COMPRESSION_INFO), null, SequentialWriterOption.DEFAULT, param, collector)) {
        for (long l = 0L; l < 1000; l++) {
            index.put(l, writer.position());
            writer.writeLong(l);
        }
        writer.finish();
    }
    CompressionMetadata comp = CompressionMetadata.create(tmp.absolutePath());
    List<SSTableReader.PartitionPositionBounds> sections = new ArrayList<>();
    for (long l : valuesToCheck) {
        long position = index.get(l);
        sections.add(new SSTableReader.PartitionPositionBounds(position, position + 8));
    }
    CompressionMetadata.Chunk[] chunks = comp.getChunksForSections(sections);
    long totalSize = comp.getTotalSizeForSections(sections);
    long expectedSize = 0;
    for (CompressionMetadata.Chunk c : chunks) expectedSize += c.length + 4;
    assertEquals(expectedSize, totalSize);
    // buffer up only relevant parts of file
    int size = 0;
    for (CompressionMetadata.Chunk c : chunks) // 4bytes CRC
    size += (c.length + 4);
    byte[] toRead = new byte[size];
    try (RandomAccessReader f = RandomAccessReader.open(tmp)) {
        int pos = 0;
        for (CompressionMetadata.Chunk c : chunks) {
            f.seek(c.offset);
            pos += f.read(toRead, pos, c.length + 4);
        }
    }
    if (testTruncate) {
        byte[] actuallyRead = new byte[50];
        System.arraycopy(toRead, 0, actuallyRead, 0, 50);
        toRead = actuallyRead;
    }
    // read buffer using CompressedInputStream
    CompressionInfo info = CompressionInfo.newInstance(chunks, param);
    if (testException) {
        testException(sections, info);
        return;
    }
    CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(toRead)), info, ChecksumType.CRC32, () -> 1.0);
    try (DataInputStream in = new DataInputStream(input)) {
        for (int i = 0; i < sections.size(); i++) {
            input.position(sections.get(i).lowerPosition);
            long readValue = in.readLong();
            assertEquals("expected " + valuesToCheck[i] + " but was " + readValue, valuesToCheck[i], readValue);
        }
    }
}