Examples with CfDef org.apache.cassandra.thrift.CfDef used on opensource projects

Example 6 with CfDef

use of org.apache.cassandra.thrift.CfDef in project titan by thinkaurelius.

the class CassandraThriftStoreManager method ensureColumnFamilyExists.

private void ensureColumnFamilyExists(String ksName, String cfName, String comparator) throws StorageException {
    CTConnection conn = null;
    try {
        KsDef keyspaceDef = ensureKeyspaceExists(ksName);
        conn = pool.borrowObject(ksName);
        Cassandra.Client client = conn.getClient();
        log.debug("Looking up metadata on keyspace {}...", ksName);
        boolean foundColumnFamily = false;
        for (CfDef cfDef : keyspaceDef.getCf_defs()) {
            String curCfName = cfDef.getName();
            if (curCfName.equals(cfName))
                foundColumnFamily = true;
        }
        if (!foundColumnFamily) {
            createColumnFamily(client, ksName, cfName, comparator);
        } else {
            log.debug("Keyspace {} and ColumnFamily {} were found.", ksName, cfName);
        }
    } catch (SchemaDisagreementException e) {
        throw new TemporaryStorageException(e);
    } catch (Exception e) {
        throw new PermanentStorageException(e);
    } finally {
        pool.returnObjectUnsafe(ksName, conn);
    }
}

Example 7 with CfDef

use of org.apache.cassandra.thrift.CfDef in project titan by thinkaurelius.

the class CassandraThriftStoreManager method clearStorage.

/**
 * Connect to Cassandra via Thrift on the specified host and port and attempt to truncate the named keyspace.
 * <p/>
 * This is a utility method intended mainly for testing. It is
 * equivalent to issuing 'truncate <cf>' for each of the column families in keyspace using
 * the cassandra-cli tool.
 * <p/>
 * Using truncate is better for a number of reasons, most significantly because it doesn't
 * involve any schema modifications which can take time to propagate across the cluster such
 * leaves nodes in the inconsistent state and could result in read/write failures.
 * Any schema modifications are discouraged until there is no traffic to Keyspace or ColumnFamilies.
 *
 * @throws StorageException if any checked Thrift or UnknownHostException is thrown in the body of this method
 */
public void clearStorage() throws StorageException {
    openStores.clear();
    // "log prefix"
    final String lp = "ClearStorage: ";
    /*
         * log4j is capable of automatically writing the name of a method that
         * generated a log message, but the docs warn that "generating caller
         * location information is extremely slow and should be avoided unless
         * execution speed is not an issue."
         */
    CTConnection conn = null;
    try {
        conn = pool.borrowObject(SYSTEM_KS);
        Cassandra.Client client = conn.getClient();
        KsDef ksDef;
        try {
            client.set_keyspace(keySpaceName);
            ksDef = client.describe_keyspace(keySpaceName);
        } catch (NotFoundException e) {
            log.debug(lp + "Keyspace {} does not exist, not attempting to truncate.", keySpaceName);
            return;
        } catch (InvalidRequestException e) {
            log.debug(lp + "InvalidRequestException when attempting to describe keyspace {}, not attempting to truncate.", keySpaceName);
            return;
        }
        if (null == ksDef) {
            log.debug(lp + "Received null KsDef for keyspace {}; not truncating its CFs", keySpaceName);
            return;
        }
        List<CfDef> cfDefs = ksDef.getCf_defs();
        if (null == cfDefs) {
            log.debug(lp + "Received empty CfDef list for keyspace {}; not truncating CFs", keySpaceName);
            return;
        }
        for (CfDef cfDef : ksDef.getCf_defs()) {
            client.truncate(cfDef.name);
            log.info(lp + "Truncated CF {} in keyspace {}", cfDef.name, keySpaceName);
        }
    /*
             * Clearing the CTConnectionPool is unnecessary. This method
             * removes no keyspaces. All open Cassandra connections will
             * remain valid.
             */
    } catch (Exception e) {
        throw new TemporaryStorageException(e);
    } finally {
        if (conn != null && conn.getClient() != null) {
            try {
                conn.getClient().set_keyspace(SYSTEM_KS);
            } catch (InvalidRequestException e) {
                log.warn("Failed to reset keyspace", e);
            } catch (TException e) {
                log.warn("Failed to reset keyspace", e);
            }
        }
        pool.returnObjectUnsafe(SYSTEM_KS, conn);
    }
}

Example 8 with CfDef

use of org.apache.cassandra.thrift.CfDef in project coprhd-controller by CoprHD.

the class SchemaUtil method checkCf.

/**
 * Checks all required CF's against keyspace definition. Any missing
 * CF's are created on the fly.
 * Note: it will require all nodes are online and converge at target Cassandra schema version
 * if parameter waitAllNodesConverge is true, otherwise, only require there's only one schema
 * version across current cluster.
 */
public void checkCf(boolean waitAllNodesConverge) throws InterruptedException, ConnectionException {
    KeyspaceDefinition kd = clientContext.getCluster().describeKeyspace(_keyspaceName);
    Cluster cluster = clientContext.getCluster();
    // Get default GC grace period for all index CFs in local DB
    Integer indexGcGrace = isGeoDbsvc() ? null : getIntProperty(DbClientImpl.DB_CASSANDRA_INDEX_GC_GRACE_PERIOD, null);
    Iterator<ColumnFamily> it = getCfMap().values().iterator();
    String latestSchemaVersion = null;
    while (it.hasNext()) {
        ColumnFamily cf = it.next();
        String comparator = cf.getColumnSerializer().getComparatorType().getTypeName();
        if (comparator.equals("CompositeType")) {
            if (cf.getColumnSerializer() instanceof CompositeColumnNameSerializer) {
                comparator = CompositeColumnNameSerializer.getComparatorName();
            } else if (cf.getColumnSerializer() instanceof IndexColumnNameSerializer) {
                comparator = IndexColumnNameSerializer.getComparatorName();
            } else if (cf.getColumnSerializer() instanceof ClassNameTimeSeriesSerializer) {
                comparator = ClassNameTimeSeriesSerializer.getComparatorName();
            } else if (cf.getColumnSerializer() instanceof TimeSeriesColumnNameSerializer) {
                comparator = TimeSeriesColumnNameSerializer.getComparatorName();
            } else {
                throw new IllegalArgumentException();
            }
        }
        ThriftColumnFamilyDefinitionImpl cfd = (ThriftColumnFamilyDefinitionImpl) kd.getColumnFamily(cf.getName());
        CfDef cdef = null;
        // The CF's gc_grace_period will be set if it's an index CF
        Integer cfGcGrace = cf.getColumnSerializer() instanceof IndexColumnNameSerializer ? indexGcGrace : null;
        // If there's specific configuration particular for this CF, take it.
        cfGcGrace = getIntProperty(DbClientImpl.DB_CASSANDRA_GC_GRACE_PERIOD_PREFIX + cf.getName(), cfGcGrace);
        if (cfd == null) {
            cfd = (ThriftColumnFamilyDefinitionImpl) cluster.makeColumnFamilyDefinition().setKeyspace(_keyspaceName).setName(cf.getName()).setComparatorType(comparator).setKeyValidationClass(cf.getKeySerializer().getComparatorType().getTypeName());
            if (_keyspaceName.equals(DbClientContext.LOCAL_KEYSPACE_NAME)) {
                cdef = cfd.getThriftColumnFamilyDefinition();
                String retry = cdef.getSpeculative_retry();
                if (!retry.equals(PERCENTILE)) {
                    try {
                        cdef.setSpeculative_retry(PERCENTILE);
                    } catch (Exception e) {
                        _log.info("Failed to set speculative_retry e=", e);
                    }
                }
            }
            TimeSeriesType tsType = TypeMap.getTimeSeriesType(cf.getName());
            if (tsType != null && tsType.getCompactOptimized() && _dbCommonInfo != null && Boolean.TRUE.toString().equalsIgnoreCase(_dbCommonInfo.getProperty(DbClientImpl.DB_STAT_OPTIMIZE_DISK_SPACE, "false"))) {
                String compactionStrategy = _dbCommonInfo.getProperty(DbClientImpl.DB_CASSANDRA_OPTIMIZED_COMPACTION_STRATEGY, "SizeTieredCompactionStrategy");
                _log.info("Setting DB compaction strategy to {}", compactionStrategy);
                int gcGrace = Integer.parseInt(_dbCommonInfo.getProperty(DbClientImpl.DB_CASSANDRA_GC_GRACE_PERIOD, // default is 10 days
                "864000"));
                _log.info("Setting DB GC grace period to {}", gcGrace);
                cfd.setCompactionStrategy(compactionStrategy).setGcGraceSeconds(gcGrace);
            } else if (cfGcGrace != null) {
                _log.info("Setting CF:{} gc_grace_period to {}", cf.getName(), cfGcGrace.intValue());
                cfd.setGcGraceSeconds(cfGcGrace.intValue());
            }
            latestSchemaVersion = addColumnFamily(cfd);
        } else {
            boolean modified = false;
            String existingComparator = cfd.getComparatorType();
            if (!matchComparator(existingComparator, comparator)) {
                _log.info("Comparator mismatch: db {} / schema {}", existingComparator, comparator);
                cfd.setComparatorType(comparator);
                modified = true;
            }
            TimeSeriesType tsType = TypeMap.getTimeSeriesType(cf.getName());
            if (tsType != null && tsType.getCompactOptimized() && _dbCommonInfo != null) {
                String compactionStrategy = _dbCommonInfo.getProperty(DbClientImpl.DB_CASSANDRA_OPTIMIZED_COMPACTION_STRATEGY, "SizeTieredCompactionStrategy");
                String existingStrategy = cfd.getCompactionStrategy();
                if (existingStrategy == null || !existingStrategy.contains(compactionStrategy)) {
                    _log.info("Setting DB compaction strategy to {}", compactionStrategy);
                    cfd.setCompactionStrategy(compactionStrategy);
                    modified = true;
                }
                int gcGrace = Integer.parseInt(_dbCommonInfo.getProperty(DbClientImpl.DB_CASSANDRA_GC_GRACE_PERIOD, "864000"));
                if (gcGrace != cfd.getGcGraceSeconds()) {
                    _log.info("Setting DB GC grace period to {}", gcGrace);
                    cfd.setGcGraceSeconds(gcGrace);
                    modified = true;
                }
            } else if (cfGcGrace != null && cfd.getGcGraceSeconds() != cfGcGrace.intValue()) {
                _log.info("Setting CF:{} gc_grace_period to {}", cf.getName(), cfGcGrace.intValue());
                cfd.setGcGraceSeconds(cfGcGrace.intValue());
                modified = true;
            }
            if (_keyspaceName.equals(DbClientContext.LOCAL_KEYSPACE_NAME)) {
                cdef = cfd.getThriftColumnFamilyDefinition();
                String retry = cdef.getSpeculative_retry();
                if (!retry.equals(PERCENTILE)) {
                    try {
                        cdef.setSpeculative_retry(PERCENTILE);
                        modified = true;
                    } catch (Exception e) {
                        _log.info("Failed to set speculative retry e=", e);
                    }
                }
            }
            if (modified) {
                latestSchemaVersion = updateColumnFamily(cfd);
            }
        }
    }
    if (latestSchemaVersion != null) {
        if (waitAllNodesConverge) {
            clientContext.waitForSchemaAgreement(latestSchemaVersion, _statusChecker.getClusterNodeCount());
        } else {
            clientContext.waitForSchemaAgreement(latestSchemaVersion, -1);
        }
    }
}

Example 9 with CfDef

use of org.apache.cassandra.thrift.CfDef in project atlasdb by palantir.

the class CassandraKeyValueServiceIntegrationTest method testCfEqualityChecker.

@Test
public void testCfEqualityChecker() throws TException {
    CassandraKeyValueServiceImpl kvs;
    if (keyValueService instanceof CassandraKeyValueService) {
        kvs = (CassandraKeyValueServiceImpl) keyValueService;
    } else if (keyValueService instanceof TableSplittingKeyValueService) {
        // scylla tests
        KeyValueService delegate = ((TableSplittingKeyValueService) keyValueService).getDelegate(testTable);
        assertTrue("The nesting of Key Value Services has apparently changed", delegate instanceof CassandraKeyValueService);
        kvs = (CassandraKeyValueServiceImpl) delegate;
    } else {
        throw new IllegalArgumentException("Can't run this cassandra-specific test against a non-cassandra KVS");
    }
    kvs.createTable(testTable, tableMetadata);
    List<CfDef> knownCfs = kvs.getClientPool().runWithRetry(client -> client.describe_keyspace("atlasdb").getCf_defs());
    CfDef clusterSideCf = Iterables.getOnlyElement(knownCfs.stream().filter(cf -> cf.getName().equals(getInternalTestTableName())).collect(Collectors.toList()));
    assertTrue("After serialization and deserialization to database, Cf metadata did not match.", ColumnFamilyDefinitions.isMatchingCf(kvs.getCfForTable(testTable, tableMetadata, FOUR_DAYS_IN_SECONDS), clusterSideCf));
}

Example 10 with CfDef

use of org.apache.cassandra.thrift.CfDef in project atlasdb by palantir.

the class CassandraKeyValueServiceImpl method upgradeFromOlderInternalSchema.

private void upgradeFromOlderInternalSchema() {
    try {
        Map<TableReference, byte[]> metadataForTables = getMetadataForTables();
        final Collection<CfDef> updatedCfs = Lists.newArrayListWithExpectedSize(metadataForTables.size());
        List<CfDef> knownCfs = clientPool.runWithRetry(client -> client.describe_keyspace(config.getKeyspaceOrThrow()).getCf_defs());
        for (CfDef clusterSideCf : knownCfs) {
            TableReference tableRef = CassandraKeyValueServices.tableReferenceFromCfDef(clusterSideCf);
            if (metadataForTables.containsKey(tableRef)) {
                byte[] clusterSideMetadata = metadataForTables.get(tableRef);
                CfDef clientSideCf = getCfForTable(tableRef, clusterSideMetadata, config.gcGraceSeconds());
                if (!ColumnFamilyDefinitions.isMatchingCf(clientSideCf, clusterSideCf)) {
                    // mismatch; we have changed how we generate schema since we last persisted
                    log.warn("Upgrading table {} to new internal Cassandra schema", LoggingArgs.tableRef(tableRef));
                    updatedCfs.add(clientSideCf);
                }
            } else if (!HiddenTables.isHidden(tableRef)) {
                // Possible to get here from a race condition with another service starting up
                // and performing schema upgrades concurrent with us doing this check
                log.error("Found a table {} that did not have persisted" + " AtlasDB metadata. If you recently did a Palantir update, try waiting until" + " schema upgrades are completed on all backend CLIs/services etc and restarting" + " this service. If this error re-occurs on subsequent attempted startups, please" + " contact Palantir support.", LoggingArgs.tableRef(tableRef));
            }
        }
        // we are racing another service to do these same operations here, but they are idempotent / safe
        Map<Cell, byte[]> emptyMetadataUpdate = ImmutableMap.of();
        if (!updatedCfs.isEmpty()) {
            putMetadataAndMaybeAlterTables(true, emptyMetadataUpdate, updatedCfs);
            log.info("New table-related settings were applied on startup!!");
        } else {
            log.info("No tables are being upgraded on startup. No updated table-related settings found.");
        }
    } catch (TException e) {
        log.error("Couldn't upgrade from an older internal Cassandra schema." + " New table-related settings may not have taken effect.");
    }
}