Examples with Descriptor org.apache.cassandra.io.sstable.Descriptor used on opensource projects

Example 11 with Descriptor

use of org.apache.cassandra.io.sstable.Descriptor in project cassandra by apache.

the class MetadataSerializerTest method testOldReadsNew.

public void testOldReadsNew(String oldV, String newV) throws IOException {
    Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
    MetadataSerializer serializer = new MetadataSerializer();
    // Write metadata in two minor formats.
    File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(newV));
    File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV));
    // Reading both as earlier version should yield identical results.
    SSTableFormat.Type stype = SSTableFormat.Type.current();
    Descriptor desc = new Descriptor(stype.info.getVersion(oldV), statsFileLb.getParentFile(), "", "", 0, stype);
    try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
        RandomAccessReader inLa = RandomAccessReader.open(statsFileLa)) {
        Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
        Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
        for (MetadataType type : MetadataType.values()) {
            assertEquals(deserializedLa.get(type), deserializedLb.get(type));
            if (!originalMetadata.get(type).equals(deserializedLb.get(type))) {
                // Currently only STATS can be different. Change if no longer the case
                assertEquals(MetadataType.STATS, type);
            }
        }
    }
}

Example 12 with Descriptor

use of org.apache.cassandra.io.sstable.Descriptor in project cassandra by apache.

the class MetadataSerializerTest method testSerialization.

@Test
public void testSerialization() throws IOException {
    Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
    MetadataSerializer serializer = new MetadataSerializer();
    File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
    Descriptor desc = new Descriptor(statsFile.getParentFile(), "", "", 0, SSTableFormat.Type.BIG);
    try (RandomAccessReader in = RandomAccessReader.open(statsFile)) {
        Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
        for (MetadataType type : MetadataType.values()) {
            assertEquals(originalMetadata.get(type), deserialized.get(type));
        }
    }
}

Example 13 with Descriptor

use of org.apache.cassandra.io.sstable.Descriptor in project cassandra by apache.

the class PerSSTableIndexWriterTest method testSparse.

@Test
public void testSparse() throws Exception {
    final String columnName = "timestamp";
    ColumnFamilyStore cfs = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME);
    ColumnMetadata column = cfs.metadata().getColumn(UTF8Type.instance.decompose(columnName));
    SASIIndex sasi = (SASIIndex) cfs.indexManager.getIndexByName(cfs.name + "_" + columnName);
    File directory = cfs.getDirectories().getDirectoryForNewSSTables();
    Descriptor descriptor = cfs.newSSTableDescriptor(directory);
    PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, OperationType.FLUSH);
    final long now = System.currentTimeMillis();
    indexWriter.begin();
    indexWriter.indexes.put(column, indexWriter.newIndex(sasi.getIndex()));
    populateSegment(cfs.metadata(), indexWriter.getIndex(column), new HashMap<Long, Set<Integer>>() {

        {
            put(now, new HashSet<>(Arrays.asList(0, 1)));
            put(now + 1, new HashSet<>(Arrays.asList(2, 3)));
            put(now + 2, new HashSet<>(Arrays.asList(4, 5, 6, 7, 8, 9)));
        }
    });
    Callable<OnDiskIndex> segmentBuilder = indexWriter.getIndex(column).scheduleSegmentFlush(false);
    Assert.assertNull(segmentBuilder.call());
    PerSSTableIndexWriter.Index index = indexWriter.getIndex(column);
    Random random = ThreadLocalRandom.current();
    Set<String> segments = new HashSet<>();
    // now let's test multiple correct segments with yield incorrect final segment
    for (int i = 0; i < 3; i++) {
        populateSegment(cfs.metadata(), index, new HashMap<Long, Set<Integer>>() {

            {
                put(now, new HashSet<>(Arrays.asList(random.nextInt(), random.nextInt(), random.nextInt())));
                put(now + 1, new HashSet<>(Arrays.asList(random.nextInt(), random.nextInt(), random.nextInt())));
                put(now + 2, new HashSet<>(Arrays.asList(random.nextInt(), random.nextInt(), random.nextInt())));
            }
        });
        try {
            // flush each of the new segments, they should all succeed
            OnDiskIndex segment = index.scheduleSegmentFlush(false).call();
            index.segments.add(Futures.immediateFuture(segment));
            segments.add(segment.getIndexPath());
        } catch (Exception | FSError e) {
            e.printStackTrace();
            Assert.fail();
        }
    }
    // make sure that all of the segments are present of the filesystem
    for (String segment : segments) Assert.assertTrue(new File(segment).exists());
    indexWriter.complete();
    // make sure that individual segments have been cleaned up
    for (String segment : segments) Assert.assertFalse(new File(segment).exists());
    // and combined index doesn't exist either
    Assert.assertFalse(new File(index.outputFile).exists());
}

Example 14 with Descriptor

use of org.apache.cassandra.io.sstable.Descriptor in project cassandra by apache.

the class MigrationManagerTest method testDropIndex.

/*
    @Test
    public void testUpdateColumnFamilyNoIndexes() throws ConfigurationException
    {
        // create a keyspace with a cf to update.
        CFMetaData cf = addTestTable("UpdatedCfKs", "Standard1added", "A new cf that will be updated");
        KSMetaData ksm = KSMetaData.testMetadata(cf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(1), cf);
        MigrationManager.announceNewKeyspace(ksm);

        assertNotNull(Schema.instance.getKSMetaData(cf.ksName));
        assertEquals(Schema.instance.getKSMetaData(cf.ksName), ksm);
        assertNotNull(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName));

        // updating certain fields should fail.
        CFMetaData newCfm = cf.copy();
        newCfm.defaultValidator(BytesType.instance);
        newCfm.minCompactionThreshold(5);
        newCfm.maxCompactionThreshold(31);

        // test valid operations.
        newCfm.comment("Modified comment");
        MigrationManager.announceTableUpdate(newCfm); // doesn't get set back here.

        newCfm.readRepairChance(0.23);
        MigrationManager.announceTableUpdate(newCfm);

        newCfm.gcGraceSeconds(12);
        MigrationManager.announceTableUpdate(newCfm);

        newCfm.defaultValidator(UTF8Type.instance);
        MigrationManager.announceTableUpdate(newCfm);

        newCfm.minCompactionThreshold(3);
        MigrationManager.announceTableUpdate(newCfm);

        newCfm.maxCompactionThreshold(33);
        MigrationManager.announceTableUpdate(newCfm);

        // can't test changing the reconciler because there is only one impl.

        // check the cumulative affect.
        assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getComment(), newCfm.getComment());
        assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getReadRepairChance(), newCfm.getReadRepairChance(), 0.0001);
        assertEquals(Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getGcGraceSeconds(), newCfm.getGcGraceSeconds());
        assertEquals(UTF8Type.instance, Schema.instance.getTableMetadataRef(cf.ksName, cf.cfName).getDefaultValidator());

        // Change tableId
        newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, cf.comparator);
        CFMetaData.copyOpts(newCfm, cf);
        try
        {
            cf.apply(newCfm);
            throw new AssertionError("Should have blown up when you used a different id.");
        }
        catch (ConfigurationException expected) {}

        // Change cfName
        newCfm = new CFMetaData(cf.ksName, cf.cfName + "_renamed", cf.cfType, cf.comparator);
        CFMetaData.copyOpts(newCfm, cf);
        try
        {
            cf.apply(newCfm);
            throw new AssertionError("Should have blown up when you used a different name.");
        }
        catch (ConfigurationException expected) {}

        // Change ksName
        newCfm = new CFMetaData(cf.ksName + "_renamed", cf.cfName, cf.cfType, cf.comparator);
        CFMetaData.copyOpts(newCfm, cf);
        try
        {
            cf.apply(newCfm);
            throw new AssertionError("Should have blown up when you used a different keyspace.");
        }
        catch (ConfigurationException expected) {}

        // Change cf type
        newCfm = new CFMetaData(cf.ksName, cf.cfName, ColumnFamilyType.Super, cf.comparator);
        CFMetaData.copyOpts(newCfm, cf);
        try
        {
            cf.apply(newCfm);
            throw new AssertionError("Should have blwon up when you used a different cf type.");
        }
        catch (ConfigurationException expected) {}

        // Change comparator
        newCfm = new CFMetaData(cf.ksName, cf.cfName, cf.cfType, new SimpleDenseCellNameType(TimeUUIDType.instance));
        CFMetaData.copyOpts(newCfm, cf);
        try
        {
            cf.apply(newCfm);
            throw new AssertionError("Should have blown up when you used a different comparator.");
        }
        catch (ConfigurationException expected) {}
    }
    */
@Test
public void testDropIndex() throws ConfigurationException {
    // persist keyspace definition in the system keyspace
    SchemaKeyspace.makeCreateKeyspaceMutation(Schema.instance.getKeyspaceMetadata(KEYSPACE6), FBUtilities.timestampMicros()).build().applyUnsafe();
    ColumnFamilyStore cfs = Keyspace.open(KEYSPACE6).getColumnFamilyStore(TABLE1i);
    String indexName = TABLE1i + "_birthdate_key_index";
    // insert some data.  save the sstable descriptor so we can make sure it's marked for delete after the drop
    QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (key, c1, birthdate, notbirthdate) VALUES (?, ?, ?, ?)", KEYSPACE6, TABLE1i), "key0", "col0", 1L, 1L);
    cfs.forceBlockingFlush();
    ColumnFamilyStore indexCfs = cfs.indexManager.getIndexByName(indexName).getBackingTable().orElseThrow(throwAssert("Cannot access index cfs"));
    Descriptor desc = indexCfs.getLiveSSTables().iterator().next().descriptor;
    // drop the index
    TableMetadata meta = cfs.metadata();
    IndexMetadata existing = meta.indexes.get(indexName).orElseThrow(throwAssert("Index not found"));
    MigrationManager.announceTableUpdate(meta.unbuild().indexes(meta.indexes.without(existing.name)).build());
    // check
    assertTrue(cfs.indexManager.listIndexes().isEmpty());
    LifecycleTransaction.waitForDeletions();
    assertFalse(new File(desc.filenameFor(Component.DATA)).exists());
}

Example 15 with Descriptor

use of org.apache.cassandra.io.sstable.Descriptor in project cassandra by apache.

the class MockSchema method sstable.

public static SSTableReader sstable(int generation, int size, boolean keepRef, ColumnFamilyStore cfs) {
    Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.keyspace.getName(), cfs.getTableName(), generation, SSTableFormat.Type.BIG);
    Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
    for (Component component : components) {
        File file = new File(descriptor.filenameFor(component));
        try {
            file.createNewFile();
        } catch (IOException e) {
        }
    }
    if (size > 0) {
        try {
            File file = new File(descriptor.filenameFor(Component.DATA));
            try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
                raf.setLength(size);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
    StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator).finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, header).get(MetadataType.STATS);
    SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata, RANDOM_ACCESS_READER_FACTORY.sharedCopy(), RANDOM_ACCESS_READER_FACTORY.sharedCopy(), indexSummary.sharedCopy(), new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header);
    reader.first = reader.last = readerBounds(generation);
    if (!keepRef)
        reader.selfRef().release();
    return reader;
}