Examples with Column com.netflix.astyanax.model.Column used on opensource projects

Example 11 with Column

use of com.netflix.astyanax.model.Column in project coprhd-controller by CoprHD.

the class InternalDbClient method migrateToGeoDb.

// only used during migration stage in single thread, so it's safe to suppress
@SuppressWarnings("findbugs:IS2_INCONSISTENT_SYNC")
public <T extends DataObject> void migrateToGeoDb(Class<T> clazz) {
    DataObjectType doType = TypeMap.getDoType(clazz);
    if (doType == null) {
        throw new IllegalArgumentException();
    }
    if (!KeyspaceUtil.isGlobal(clazz)) {
        throw new IllegalArgumentException(String.format("CF %s is not a global resource", clazz.getName()));
    }
    // this CF ensured to be a global resource
    doType.setEncryptionProvider(_geoEncryptionProvider);
    // find all the records of CF <T> in local db, similar to queryByType(clazz, false)
    URIQueryResultList result = new URIQueryResultList();
    DecommissionedConstraint constraint = DecommissionedConstraint.Factory.getAllObjectsConstraint(clazz, null);
    constraint.setKeyspace(localContext.getKeyspace());
    constraint.execute(result);
    Iterator<URI> recIt = result.iterator();
    List<URI> batch = getNextBatch(recIt);
    while (!batch.isEmpty()) {
        Rows<String, CompositeColumnName> rows = queryRowsWithAllColumns(localContext.getKeyspace(), batch, doType.getCF());
        Iterator<Row<String, CompositeColumnName>> it = rows.iterator();
        while (it.hasNext()) {
            Row<String, CompositeColumnName> row = it.next();
            try {
                if (row.getColumns().size() == 0) {
                    continue;
                }
                // can't simply use doType.deserialize(clazz, row, cleanList) below
                // since the DataObject instance retrieved in this way doesn't have
                // change tracking information within and nothing gets persisted into
                // db in the end.
                log.info("Migrating record {} to geo db", row.getKey());
                DataObject obj = DataObject.createInstance(clazz, URI.create(row.getKey()));
                obj.trackChanges();
                Iterator<Column<CompositeColumnName>> columnIterator = row.getColumns().iterator();
                while (columnIterator.hasNext()) {
                    Column<CompositeColumnName> column = columnIterator.next();
                    ColumnField columnField = doType.getColumnField(column.getName().getOne());
                    if (columnField.isEncrypted()) {
                        // Decrypt using the local encryption provider and later
                        // encrypt it again using the geo encryption provider
                        columnField.deserializeEncryptedColumn(column, obj, _encryptionProvider);
                    } else {
                        columnField.deserialize(column, obj);
                    }
                    // set changed for ChangeTracking structures
                    columnField.setChanged(obj);
                }
                // persist the object into geo db, similar to createObject(objects)
                // only that we need to specify the keyspace explicitly here
                // also we shouldn't overwrite the creation time
                boolean retryFailedWriteWithLocalQuorum = shouldRetryFailedWriteWithLocalQuorum(clazz);
                RowMutator mutator = new RowMutator(geoContext.getKeyspace(), retryFailedWriteWithLocalQuorum);
                doType.serialize(mutator, obj);
                mutator.execute();
            } catch (final InstantiationException e) {
                throw DatabaseException.fatals.queryFailed(e);
            } catch (final IllegalAccessException e) {
                throw DatabaseException.fatals.queryFailed(e);
            }
        }
        batch = getNextBatch(recIt);
    }
}

Example 12 with Column

use of com.netflix.astyanax.model.Column in project coprhd-controller by CoprHD.

the class DbConsistencyCheckerHelper method checkCFIndices.

/**
 * Scan all the data object records, to find out the object record is existing but the related index is missing.
 *
 * @param doType
 * @param toConsole whether print out in the console
 * @return the number of corrupted data
 * @throws ConnectionException
 */
public void checkCFIndices(DataObjectType doType, boolean toConsole, CheckResult checkResult) throws ConnectionException {
    initSchemaVersions();
    Class objClass = doType.getDataObjectClass();
    if (skipCheckCFs.contains(objClass.getSimpleName())) {
        _log.info("Skip checking CF {}", objClass);
        return;
    } else {
        _log.info("Check Data Object CF {} with double confirmed option: {}", objClass, doubleConfirmed);
    }
    Map<String, ColumnField> indexedFields = new HashMap<String, ColumnField>();
    for (ColumnField field : doType.getColumnFields()) {
        if (field.getIndex() != null) {
            indexedFields.put(field.getName(), field);
        }
    }
    if (indexedFields.isEmpty()) {
        return;
    }
    Keyspace keyspace = dbClient.getKeyspace(objClass);
    ColumnFamilyQuery<String, CompositeColumnName> query = keyspace.prepareQuery(doType.getCF());
    OperationResult<Rows<String, CompositeColumnName>> result = query.getAllRows().setRowLimit(dbClient.DEFAULT_PAGE_SIZE).execute();
    int scannedRows = 0;
    long beginTime = System.currentTimeMillis();
    for (Row<String, CompositeColumnName> objRow : result.getResult()) {
        try {
            boolean inactiveObject = false;
            boolean hasStaleEntries = false;
            scannedRows++;
            Map<String, Column<CompositeColumnName>> distinctColumns = new HashMap<String, Column<CompositeColumnName>>();
            for (Column<CompositeColumnName> column : objRow.getColumns()) {
                distinctColumns.put(column.getName().getOne(), column);
            }
            hasStaleEntries = checkForStaleEntries(distinctColumns);
            if (hasStaleEntries) {
                _log.warn("Data object with key {} has stale entries, don't rebuild index for it.", objRow.getKey());
                continue;
            }
            for (Column<CompositeColumnName> column : objRow.getColumns()) {
                if (column.getName().getOne().equals(DataObject.INACTIVE_FIELD_NAME)) {
                    inactiveObject = column.getBooleanValue();
                    break;
                }
            }
            if (inactiveObject) {
                continue;
            }
            for (Column<CompositeColumnName> column : objRow.getColumns()) {
                if (!indexedFields.containsKey(column.getName().getOne())) {
                    continue;
                }
                // we don't build index if the value is null, refer to ColumnField.
                if (!column.hasValue()) {
                    continue;
                }
                ColumnField indexedField = indexedFields.get(column.getName().getOne());
                String indexKey = getIndexKey(indexedField, column, objRow);
                if (indexKey == null) {
                    continue;
                }
                boolean isColumnInIndex = isColumnInIndex(keyspace, indexedField.getIndexCF(), indexKey, getIndexColumns(indexedField, column, objRow.getKey()));
                if (!isColumnInIndex) {
                    if (doubleConfirmed && isDataObjectRemoved(doType.getDataObjectClass(), objRow.getKey())) {
                        continue;
                    }
                    String dbVersion = findDataCreatedInWhichDBVersion(column.getName().getTimeUUID());
                    checkResult.increaseByVersion(dbVersion);
                    logMessage(String.format("Inconsistency found Object(%s, id: %s, field: %s) is existing, but the related Index(%s, type: %s, id: %s) is missing. This entry is updated by version %s", indexedField.getDataObjectType().getSimpleName(), objRow.getKey(), indexedField.getName(), indexedField.getIndexCF().getName(), indexedField.getIndex().getClass().getSimpleName(), indexKey, dbVersion), true, toConsole);
                    DbCheckerFileWriter.writeTo(DbCheckerFileWriter.WRITER_REBUILD_INDEX, String.format("id:%s, cfName:%s", objRow.getKey(), doType.getCF().getName()));
                }
            }
            if (scannedRows >= THRESHHOLD_FOR_OUTPUT_DEBUG) {
                _log.info("{} data objects have been check with time {}", scannedRows, DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - beginTime));
                scannedRows = 0;
                beginTime = System.currentTimeMillis();
            }
        } catch (Exception e) {
            _log.warn("exception occurs when checking CF indexes", e);
        }
    }
}

Example 13 with Column

use of com.netflix.astyanax.model.Column in project coprhd-controller by CoprHD.

the class AggregateDbIndex method removeColumn.

@Override
boolean removeColumn(String recordKey, Column<CompositeColumnName> column, String className, RowMutator mutator, Map<String, List<Column<CompositeColumnName>>> fieldColumnMap) {
    IndexColumnName indexField = new IndexColumnName(fieldName, recordKey, column.getName().getTimeUUID());
    if (groupGlobal) {
        mutator.getIndexColumnList(indexCF, className).deleteColumn(indexField);
    }
    for (String group : groupBy) {
        ColumnField field = groupByFields.get(group);
        List<Column<CompositeColumnName>> groupByColumns = fieldColumnMap.get(field.getName());
        if (groupByColumns != null) {
            for (Column<CompositeColumnName> groupByCol : groupByColumns) {
                Object groupValue = ColumnValue.getPrimitiveColumnValue(groupByCol, field.getPropertyDescriptor());
                if (groupValue != null) {
                    mutator.getIndexColumnList(indexCF, getRowKey(className, groupValue)).deleteColumn(indexField);
                }
            }
        }
    }
    return true;
}

Example 14 with Column

use of com.netflix.astyanax.model.Column in project coprhd-controller by CoprHD.

the class AggregateDbIndex method removeColumn.

@Override
boolean removeColumn(String recordKey, Column<CompositeColumnName> column, String className, RowMutator mutator, Map<String, List<Column<CompositeColumnName>>> fieldColumnMap, DataObject obj) {
    IndexColumnName indexField = new IndexColumnName(fieldName, recordKey, (UUID) null);
    for (String group : groupBy) {
        ColumnField field = groupByFields.get(group);
        List<Column<CompositeColumnName>> groupByColumns = fieldColumnMap.get(field.getName());
        if (groupByColumns != null) {
            Object latestValue = null;
            if (obj != null) {
                latestValue = ColumnField.getFieldValue(field, obj);
            }
            for (Column<CompositeColumnName> groupByCol : groupByColumns) {
                Object groupValue = ColumnValue.getPrimitiveColumnValue(groupByCol, field.getPropertyDescriptor());
                if (groupValue != null && !groupValue.equals(latestValue)) {
                    mutator.getIndexColumnList(indexCF, getRowKey(className, groupValue)).deleteColumn(indexField);
                }
            }
        }
    }
    return true;
}

Example 15 with Column

use of com.netflix.astyanax.model.Column in project coprhd-controller by CoprHD.

the class InternalDbClient method removeFieldIndex.

public <T extends DataObject> void removeFieldIndex(Class<T> clazz, String fieldName, String indexCf) {
    log.debug("removing index records for " + clazz.getSimpleName() + ":" + fieldName + " from index cf table: " + indexCf);
    DataObjectType doType = TypeMap.getDoType(clazz);
    if (doType == null) {
        throw new IllegalArgumentException();
    }
    ColumnField columnField = doType.getColumnField(fieldName);
    if (columnField == null) {
        throw new IllegalArgumentException();
    }
    List<URI> allrecs = queryByType(clazz, false);
    Keyspace ks = getKeyspace(clazz);
    Iterator<URI> recIt = allrecs.iterator();
    List<URI> batch = getNextBatch(recIt);
    while (!batch.isEmpty()) {
        Rows<String, CompositeColumnName> rows = queryRowsWithAColumn(ks, batch, doType.getCF(), columnField);
        Iterator<Row<String, CompositeColumnName>> it = rows.iterator();
        Map<String, List<Column<CompositeColumnName>>> removeList = new HashMap<String, List<Column<CompositeColumnName>>>();
        while (it.hasNext()) {
            Row<String, CompositeColumnName> row = it.next();
            if (row.getColumns().size() == 0) {
                continue;
            }
            Iterator<Column<CompositeColumnName>> columnIterator = row.getColumns().iterator();
            while (columnIterator.hasNext()) {
                Column<CompositeColumnName> column = columnIterator.next();
                if (removeList.get(row.getKey()) == null) {
                    removeList.put(row.getKey(), new ArrayList<Column<CompositeColumnName>>());
                }
                removeList.get(row.getKey()).add(column);
            }
        }
        boolean retryFailedWriteWithLocalQuorum = shouldRetryFailedWriteWithLocalQuorum(clazz);
        RowMutator mutator = new RowMutator(ks, retryFailedWriteWithLocalQuorum);
        _indexCleaner.removeOldIndex(mutator, doType, removeList, indexCf);
        batch = getNextBatch(recIt);
    }
}