Search in sources :

Example 1 with Indexes

use of org.apache.cassandra.schema.Indexes in project cassandra by apache.

the class AlterTableStatement method announceMigration.

public Event.SchemaChange announceMigration(QueryState queryState, boolean isLocalOnly) throws RequestValidationException {
    TableMetadata current = Schema.instance.validateTable(keyspace(), columnFamily());
    if (current.isView())
        throw new InvalidRequestException("Cannot use ALTER TABLE on Materialized View");
    TableMetadata.Builder builder = current.unbuild();
    ColumnIdentifier columnName = null;
    ColumnMetadata def = null;
    CQL3Type.Raw dataType = null;
    boolean isStatic = false;
    CQL3Type validator = null;
    List<ViewMetadata> viewUpdates = new ArrayList<>();
    Iterable<ViewMetadata> views = View.findAll(keyspace(), columnFamily());
    switch(oType) {
        case ALTER:
            throw new InvalidRequestException("Altering of types is not allowed");
        case ADD:
            if (current.isDense())
                throw new InvalidRequestException("Cannot add new column to a COMPACT STORAGE table");
            for (AlterTableStatementColumn colData : colNameList) {
                columnName = colData.getColumnName().getIdentifier(current);
                def = builder.getColumn(columnName);
                dataType = colData.getColumnType();
                assert dataType != null;
                isStatic = colData.getStaticType();
                validator = dataType.prepare(keyspace());
                if (isStatic) {
                    if (!current.isCompound())
                        throw new InvalidRequestException("Static columns are not allowed in COMPACT STORAGE tables");
                    if (current.clusteringColumns().isEmpty())
                        throw new InvalidRequestException("Static columns are only useful (and thus allowed) if the table has at least one clustering column");
                }
                if (def != null) {
                    switch(def.kind) {
                        case PARTITION_KEY:
                        case CLUSTERING:
                            throw new InvalidRequestException(String.format("Invalid column name %s because it conflicts with a PRIMARY KEY part", columnName));
                        default:
                            throw new InvalidRequestException(String.format("Invalid column name %s because it conflicts with an existing column", columnName));
                    }
                }
                // Cannot re-add a dropped counter column. See #7831.
                if (current.isCounter() && current.getDroppedColumn(columnName.bytes) != null)
                    throw new InvalidRequestException(String.format("Cannot re-add previously dropped counter column %s", columnName));
                AbstractType<?> type = validator.getType();
                if (type.isCollection() && type.isMultiCell()) {
                    if (!current.isCompound())
                        throw new InvalidRequestException("Cannot use non-frozen collections in COMPACT STORAGE tables");
                    if (current.isSuper())
                        throw new InvalidRequestException("Cannot use non-frozen collections with super column families");
                    // If there used to be a non-frozen collection column with the same name (that has been dropped),
                    // we could still have some data using the old type, and so we can't allow adding a collection
                    // with the same name unless the types are compatible (see #6276).
                    DroppedColumn dropped = current.droppedColumns.get(columnName.bytes);
                    if (dropped != null && dropped.column.type instanceof CollectionType && dropped.column.type.isMultiCell() && !type.isCompatibleWith(dropped.column.type)) {
                        String message = String.format("Cannot add a collection with the name %s because a collection with the same name" + " and a different type (%s) has already been used in the past", columnName, dropped.column.type.asCQL3Type());
                        throw new InvalidRequestException(message);
                    }
                }
                builder.addColumn(isStatic ? ColumnMetadata.staticColumn(current, columnName.bytes, type) : ColumnMetadata.regularColumn(current, columnName.bytes, type));
                // as well
                if (!isStatic)
                    for (ViewMetadata view : views) if (view.includeAllColumns)
                        viewUpdates.add(view.withAddedRegularColumn(ColumnMetadata.regularColumn(view.metadata, columnName.bytes, type)));
            }
            break;
        case DROP:
            if (!current.isCQLTable())
                throw new InvalidRequestException("Cannot drop columns from a non-CQL3 table");
            for (AlterTableStatementColumn colData : colNameList) {
                columnName = colData.getColumnName().getIdentifier(current);
                def = builder.getColumn(columnName);
                if (def == null)
                    throw new InvalidRequestException(String.format("Column %s was not found in table %s", columnName, columnFamily()));
                switch(def.kind) {
                    case PARTITION_KEY:
                    case CLUSTERING:
                        throw new InvalidRequestException(String.format("Cannot drop PRIMARY KEY part %s", columnName));
                    case REGULAR:
                    case STATIC:
                        builder.removeRegularOrStaticColumn(def.name);
                        builder.recordColumnDrop(def, deleteTimestamp == null ? queryState.getTimestamp() : deleteTimestamp);
                        break;
                }
                // If the dropped column is required by any secondary indexes
                // we reject the operation, as the indexes must be dropped first
                Indexes allIndexes = current.indexes;
                if (!allIndexes.isEmpty()) {
                    ColumnFamilyStore store = Keyspace.openAndGetStore(current);
                    Set<IndexMetadata> dependentIndexes = store.indexManager.getDependentIndexes(def);
                    if (!dependentIndexes.isEmpty()) {
                        throw new InvalidRequestException(String.format("Cannot drop column %s because it has " + "dependent secondary indexes (%s)", def, dependentIndexes.stream().map(i -> i.name).collect(Collectors.joining(","))));
                    }
                }
                // If a column is dropped which is included in a view, we don't allow the drop to take place.
                boolean rejectAlter = false;
                StringBuilder viewNames = new StringBuilder();
                for (ViewMetadata view : views) {
                    if (!view.includes(columnName))
                        continue;
                    if (rejectAlter)
                        viewNames.append(',');
                    rejectAlter = true;
                    viewNames.append(view.name);
                }
                if (rejectAlter)
                    throw new InvalidRequestException(String.format("Cannot drop column %s, depended on by materialized views (%s.{%s})", columnName.toString(), keyspace(), viewNames.toString()));
            }
            break;
        case OPTS:
            if (attrs == null)
                throw new InvalidRequestException("ALTER TABLE WITH invoked, but no parameters found");
            attrs.validate();
            TableParams params = attrs.asAlteredTableParams(current.params);
            if (!Iterables.isEmpty(views) && params.gcGraceSeconds == 0) {
                throw new InvalidRequestException("Cannot alter gc_grace_seconds of the base table of a " + "materialized view to 0, since this value is used to TTL " + "undelivered updates. Setting gc_grace_seconds too low might " + "cause undelivered updates to expire " + "before being replayed.");
            }
            if (current.isCounter() && params.defaultTimeToLive > 0)
                throw new InvalidRequestException("Cannot set default_time_to_live on a table with counters");
            builder.params(params);
            break;
        case RENAME:
            for (Map.Entry<ColumnMetadata.Raw, ColumnMetadata.Raw> entry : renames.entrySet()) {
                ColumnIdentifier from = entry.getKey().getIdentifier(current);
                ColumnIdentifier to = entry.getValue().getIdentifier(current);
                def = current.getColumn(from);
                if (def == null)
                    throw new InvalidRequestException(String.format("Cannot rename unknown column %s in table %s", from, current.name));
                if (current.getColumn(to) != null)
                    throw new InvalidRequestException(String.format("Cannot rename column %s to %s in table %s; another column of that name already exist", from, to, current.name));
                if (!def.isPrimaryKeyColumn())
                    throw new InvalidRequestException(String.format("Cannot rename non PRIMARY KEY part %s", from));
                if (!current.indexes.isEmpty()) {
                    ColumnFamilyStore store = Keyspace.openAndGetStore(current);
                    Set<IndexMetadata> dependentIndexes = store.indexManager.getDependentIndexes(def);
                    if (!dependentIndexes.isEmpty())
                        throw new InvalidRequestException(String.format("Cannot rename column %s because it has " + "dependent secondary indexes (%s)", from, dependentIndexes.stream().map(i -> i.name).collect(Collectors.joining(","))));
                }
                builder.renamePrimaryKeyColumn(from, to);
                // If the view includes a renamed column, it must be renamed in the view table and the definition.
                for (ViewMetadata view : views) {
                    if (!view.includes(from))
                        continue;
                    ColumnIdentifier viewFrom = entry.getKey().getIdentifier(view.metadata);
                    ColumnIdentifier viewTo = entry.getValue().getIdentifier(view.metadata);
                    viewUpdates.add(view.renamePrimaryKeyColumn(viewFrom, viewTo));
                }
            }
            break;
    }
    // FIXME: Should really be a single announce for the table and views.
    MigrationManager.announceTableUpdate(builder.build(), isLocalOnly);
    for (ViewMetadata viewUpdate : viewUpdates) MigrationManager.announceViewUpdate(viewUpdate, isLocalOnly);
    return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
}
Also used : DroppedColumn(org.apache.cassandra.schema.DroppedColumn) TableParams(org.apache.cassandra.schema.TableParams) java.util(java.util) Iterables(com.google.common.collect.Iterables) ColumnMetadata(org.apache.cassandra.schema.ColumnMetadata) IndexMetadata(org.apache.cassandra.schema.IndexMetadata) QueryState(org.apache.cassandra.service.QueryState) Permission(org.apache.cassandra.auth.Permission) ClientState(org.apache.cassandra.service.ClientState) AbstractType(org.apache.cassandra.db.marshal.AbstractType) Collectors(java.util.stream.Collectors) ViewMetadata(org.apache.cassandra.schema.ViewMetadata) org.apache.cassandra.cql3(org.apache.cassandra.cql3) Indexes(org.apache.cassandra.schema.Indexes) Schema(org.apache.cassandra.schema.Schema) View(org.apache.cassandra.db.view.View) MigrationManager(org.apache.cassandra.schema.MigrationManager) ColumnFamilyStore(org.apache.cassandra.db.ColumnFamilyStore) org.apache.cassandra.exceptions(org.apache.cassandra.exceptions) TableMetadata(org.apache.cassandra.schema.TableMetadata) Keyspace(org.apache.cassandra.db.Keyspace) CollectionType(org.apache.cassandra.db.marshal.CollectionType) Event(org.apache.cassandra.transport.Event) ColumnMetadata(org.apache.cassandra.schema.ColumnMetadata) CollectionType(org.apache.cassandra.db.marshal.CollectionType) DroppedColumn(org.apache.cassandra.schema.DroppedColumn) IndexMetadata(org.apache.cassandra.schema.IndexMetadata) ViewMetadata(org.apache.cassandra.schema.ViewMetadata) TableMetadata(org.apache.cassandra.schema.TableMetadata) TableParams(org.apache.cassandra.schema.TableParams) Indexes(org.apache.cassandra.schema.Indexes) ColumnFamilyStore(org.apache.cassandra.db.ColumnFamilyStore)

Example 2 with Indexes

use of org.apache.cassandra.schema.Indexes in project cassandra by apache.

the class CustomIndexTest method assertIndexCreated.

private void assertIndexCreated(String name, Map<String, String> options, List<IndexTarget> targets) {
    // all tests here use StubIndex as the custom index class,
    // so add that to the map of options
    options.put(CUSTOM_INDEX_OPTION_NAME, StubIndex.class.getName());
    IndexMetadata expected = IndexMetadata.fromIndexTargets(targets, name, IndexMetadata.Kind.CUSTOM, options);
    Indexes indexes = getCurrentColumnFamilyStore().metadata().indexes;
    for (IndexMetadata actual : indexes) if (actual.equals(expected))
        return;
    fail(String.format("Index %s not found", expected));
}
Also used : IndexMetadata(org.apache.cassandra.schema.IndexMetadata) Indexes(org.apache.cassandra.schema.Indexes)

Example 3 with Indexes

use of org.apache.cassandra.schema.Indexes in project cassandra by apache.

the class SecondaryIndexManager method reload.

/**
     * Drops and adds new indexes associated with the underlying CF
     */
public void reload() {
    // figure out what needs to be added and dropped.
    Indexes tableIndexes = baseCfs.metadata().indexes;
    indexes.keySet().stream().filter(indexName -> !tableIndexes.has(indexName)).forEach(this::removeIndex);
    // some may not have been created here yet, only added to schema
    for (IndexMetadata tableIndex : tableIndexes) addIndex(tableIndex);
}
Also used : java.util(java.util) Iterables(com.google.common.collect.Iterables) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) JMXEnabledThreadPoolExecutor(org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor) CompactionManager(org.apache.cassandra.db.compaction.CompactionManager) ColumnMetadata(org.apache.cassandra.schema.ColumnMetadata) SSTableSet(org.apache.cassandra.db.lifecycle.SSTableSet) LoggerFactory(org.slf4j.LoggerFactory) org.apache.cassandra.db(org.apache.cassandra.db) Constructor(java.lang.reflect.Constructor) Function(java.util.function.Function) StringUtils(org.apache.commons.lang3.StringUtils) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) Indexes(org.apache.cassandra.schema.Indexes) org.apache.cassandra.db.rows(org.apache.cassandra.db.rows) OpOrder(org.apache.cassandra.utils.concurrent.OpOrder) Strings(com.google.common.base.Strings) org.apache.cassandra.index.transactions(org.apache.cassandra.index.transactions) IndexTarget(org.apache.cassandra.cql3.statements.IndexTarget) ProtocolVersion(org.apache.cassandra.transport.ProtocolVersion) Refs(org.apache.cassandra.utils.concurrent.Refs) DatabaseDescriptor(org.apache.cassandra.config.DatabaseDescriptor) InvalidRequestException(org.apache.cassandra.exceptions.InvalidRequestException) org.apache.cassandra.db.partitions(org.apache.cassandra.db.partitions) Longs(com.google.common.primitives.Longs) ImmutableSet(com.google.common.collect.ImmutableSet) Logger(org.slf4j.Logger) RowFilter(org.apache.cassandra.db.filter.RowFilter) FBUtilities(org.apache.cassandra.utils.FBUtilities) IndexMetadata(org.apache.cassandra.schema.IndexMetadata) java.util.concurrent(java.util.concurrent) Tracing(org.apache.cassandra.tracing.Tracing) Collectors(java.util.stream.Collectors) Maps(com.google.common.collect.Maps) Sets(com.google.common.collect.Sets) Futures(com.google.common.util.concurrent.Futures) NamedThreadFactory(org.apache.cassandra.concurrent.NamedThreadFactory) CassandraIndex(org.apache.cassandra.index.internal.CassandraIndex) Stream(java.util.stream.Stream) SinglePartitionPager(org.apache.cassandra.service.pager.SinglePartitionPager) StageManager(org.apache.cassandra.concurrent.StageManager) Joiner(com.google.common.base.Joiner) View(org.apache.cassandra.db.lifecycle.View) Indexes(org.apache.cassandra.schema.Indexes) IndexMetadata(org.apache.cassandra.schema.IndexMetadata)

Example 4 with Indexes

use of org.apache.cassandra.schema.Indexes in project cassandra by apache.

the class SecondaryIndexManager method indexPartition.

/**
     * When building an index against existing data in sstables, add the given partition to the index
     */
public void indexPartition(DecoratedKey key, Set<Index> indexes, int pageSize) {
    if (logger.isTraceEnabled())
        logger.trace("Indexing partition {}", baseCfs.metadata().partitionKeyType.getString(key.getKey()));
    if (!indexes.isEmpty()) {
        SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(baseCfs.metadata(), FBUtilities.nowInSeconds(), key);
        int nowInSec = cmd.nowInSec();
        boolean readStatic = false;
        SinglePartitionPager pager = new SinglePartitionPager(cmd, null, ProtocolVersion.CURRENT);
        while (!pager.isExhausted()) {
            try (ReadExecutionController controller = cmd.executionController();
                OpOrder.Group writeGroup = Keyspace.writeOrder.start();
                UnfilteredPartitionIterator page = pager.fetchPageUnfiltered(baseCfs.metadata(), pageSize, controller)) {
                if (!page.hasNext())
                    break;
                try (UnfilteredRowIterator partition = page.next()) {
                    Set<Index.Indexer> indexers = indexes.stream().map(index -> index.indexerFor(key, partition.columns(), nowInSec, writeGroup, IndexTransaction.Type.UPDATE)).filter(Objects::nonNull).collect(Collectors.toSet());
                    // Short-circuit empty partitions if static row is processed or isn't read
                    if (!readStatic && partition.isEmpty() && partition.staticRow().isEmpty())
                        break;
                    indexers.forEach(Index.Indexer::begin);
                    if (!readStatic) {
                        if (!partition.staticRow().isEmpty())
                            indexers.forEach(indexer -> indexer.insertRow(partition.staticRow()));
                        indexers.forEach((Index.Indexer i) -> i.partitionDelete(partition.partitionLevelDeletion()));
                        readStatic = true;
                    }
                    MutableDeletionInfo.Builder deletionBuilder = MutableDeletionInfo.builder(partition.partitionLevelDeletion(), baseCfs.getComparator(), false);
                    while (partition.hasNext()) {
                        Unfiltered unfilteredRow = partition.next();
                        if (unfilteredRow.isRow()) {
                            Row row = (Row) unfilteredRow;
                            indexers.forEach(indexer -> indexer.insertRow(row));
                        } else {
                            assert unfilteredRow.isRangeTombstoneMarker();
                            RangeTombstoneMarker marker = (RangeTombstoneMarker) unfilteredRow;
                            deletionBuilder.add(marker);
                        }
                    }
                    MutableDeletionInfo deletionInfo = deletionBuilder.build();
                    if (deletionInfo.hasRanges()) {
                        Iterator<RangeTombstone> iter = deletionInfo.rangeIterator(false);
                        while (iter.hasNext()) indexers.forEach(indexer -> indexer.rangeTombstone(iter.next()));
                    }
                    indexers.forEach(Index.Indexer::finish);
                }
            }
        }
    }
}
Also used : java.util(java.util) Iterables(com.google.common.collect.Iterables) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) JMXEnabledThreadPoolExecutor(org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor) CompactionManager(org.apache.cassandra.db.compaction.CompactionManager) ColumnMetadata(org.apache.cassandra.schema.ColumnMetadata) SSTableSet(org.apache.cassandra.db.lifecycle.SSTableSet) LoggerFactory(org.slf4j.LoggerFactory) org.apache.cassandra.db(org.apache.cassandra.db) Constructor(java.lang.reflect.Constructor) Function(java.util.function.Function) StringUtils(org.apache.commons.lang3.StringUtils) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) Indexes(org.apache.cassandra.schema.Indexes) org.apache.cassandra.db.rows(org.apache.cassandra.db.rows) OpOrder(org.apache.cassandra.utils.concurrent.OpOrder) Strings(com.google.common.base.Strings) org.apache.cassandra.index.transactions(org.apache.cassandra.index.transactions) IndexTarget(org.apache.cassandra.cql3.statements.IndexTarget) ProtocolVersion(org.apache.cassandra.transport.ProtocolVersion) Refs(org.apache.cassandra.utils.concurrent.Refs) DatabaseDescriptor(org.apache.cassandra.config.DatabaseDescriptor) InvalidRequestException(org.apache.cassandra.exceptions.InvalidRequestException) org.apache.cassandra.db.partitions(org.apache.cassandra.db.partitions) Longs(com.google.common.primitives.Longs) ImmutableSet(com.google.common.collect.ImmutableSet) Logger(org.slf4j.Logger) RowFilter(org.apache.cassandra.db.filter.RowFilter) FBUtilities(org.apache.cassandra.utils.FBUtilities) IndexMetadata(org.apache.cassandra.schema.IndexMetadata) java.util.concurrent(java.util.concurrent) Tracing(org.apache.cassandra.tracing.Tracing) Collectors(java.util.stream.Collectors) Maps(com.google.common.collect.Maps) Sets(com.google.common.collect.Sets) Futures(com.google.common.util.concurrent.Futures) NamedThreadFactory(org.apache.cassandra.concurrent.NamedThreadFactory) CassandraIndex(org.apache.cassandra.index.internal.CassandraIndex) Stream(java.util.stream.Stream) SinglePartitionPager(org.apache.cassandra.service.pager.SinglePartitionPager) StageManager(org.apache.cassandra.concurrent.StageManager) Joiner(com.google.common.base.Joiner) View(org.apache.cassandra.db.lifecycle.View) SinglePartitionPager(org.apache.cassandra.service.pager.SinglePartitionPager) OpOrder(org.apache.cassandra.utils.concurrent.OpOrder)

Aggregations

IndexMetadata (org.apache.cassandra.schema.IndexMetadata)4 Indexes (org.apache.cassandra.schema.Indexes)4 Iterables (com.google.common.collect.Iterables)3 java.util (java.util)3 Collectors (java.util.stream.Collectors)3 ColumnMetadata (org.apache.cassandra.schema.ColumnMetadata)3 Joiner (com.google.common.base.Joiner)2 Strings (com.google.common.base.Strings)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Maps (com.google.common.collect.Maps)2 Sets (com.google.common.collect.Sets)2 Longs (com.google.common.primitives.Longs)2 Futures (com.google.common.util.concurrent.Futures)2 MoreExecutors (com.google.common.util.concurrent.MoreExecutors)2 Constructor (java.lang.reflect.Constructor)2 java.util.concurrent (java.util.concurrent)2 Function (java.util.function.Function)2 Stream (java.util.stream.Stream)2 JMXEnabledThreadPoolExecutor (org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor)2 NamedThreadFactory (org.apache.cassandra.concurrent.NamedThreadFactory)2