Examples with Keyspace com.netflix.astyanax.Keyspace used on opensource projects

Example 21 with Keyspace

use of com.netflix.astyanax.Keyspace in project coprhd-controller by CoprHD.

the class DbConsistencyCheckerHelperTest method testCheckCFIndexing.

@Test
public void testCheckCFIndexing() throws Exception {
    ColumnFamily<String, CompositeColumnName> cf = new ColumnFamily<String, CompositeColumnName>("FileShare", StringSerializer.get(), CompositeColumnNameSerializer.get());
    ColumnFamily<String, IndexColumnName> indexCF = new ColumnFamily<String, IndexColumnName>("AltIdIndex", StringSerializer.get(), IndexColumnNameSerializer.get());
    Keyspace keyspace = ((DbClientImpl) getDbClient()).getLocalContext().getKeyspace();
    FileShare testData = new FileShare();
    testData.setId(URIUtil.createId(FileShare.class));
    testData.setPath("A1");
    testData.setMountPath("A2");
    getDbClient().createObject(testData);
    keyspace.prepareQuery(indexCF).withCql(String.format("delete from \"AltIdIndex\" where key='%s'", "A1")).execute();
    CheckResult checkResult = new CheckResult();
    helper.checkCFIndices(TypeMap.getDoType(FileShare.class), false, checkResult);
    assertEquals(1, checkResult.getTotal());
    keyspace.prepareQuery(indexCF).withCql(String.format("delete from \"AltIdIndex\" where key='%s'", "A2")).execute();
    checkResult = new CheckResult();
    helper.checkCFIndices(TypeMap.getDoType(FileShare.class), false, checkResult);
    assertEquals(2, checkResult.getTotal());
    helper = new DbConsistencyCheckerHelper((DbClientImpl) getDbClient()) {

        @Override
        protected boolean isDataObjectRemoved(Class<? extends DataObject> clazz, String key) {
            return true;
        }
    };
    checkResult = new CheckResult();
    helper.checkCFIndices(TypeMap.getDoType(FileShare.class), false, checkResult);
    assertEquals(0, checkResult.getTotal());
    testData = new FileShare();
    testData.setId(URIUtil.createId(FileShare.class));
    testData.setPath("A'A'");
    testData.setMountPath("A2");
    getDbClient().createObject(testData);
    checkResult = new CheckResult();
    helper.checkCFIndices(TypeMap.getDoType(FileShare.class), false, checkResult);
    assertEquals(0, checkResult.getTotal());
}

Example 22 with Keyspace

use of com.netflix.astyanax.Keyspace in project coprhd-controller by CoprHD.

the class DbConsistencyCheckerHelperTest method testClassNameTimeSeriesIndex.

@Test
public void testClassNameTimeSeriesIndex() throws Exception {
    DbConsistencyCheckerHelperMock mockHelper = new DbConsistencyCheckerHelperMock((DbClientImpl) getDbClient());
    Order order = new Order();
    order.setId(URIUtil.createId(Order.class));
    order.setLabel("order1");
    order.setSubmittedByUserId("root");
    getDbClient().createObject(order);
    Keyspace keyspace = ((DbClientImpl) getDbClient()).getLocalContext().getKeyspace();
    ColumnFamily<String, ClassNameTimeSeriesIndexColumnName> indexCF = new ColumnFamily<String, ClassNameTimeSeriesIndexColumnName>("UserToOrdersByTimeStamp", StringSerializer.get(), ClassNameTimeSeriesSerializer.get());
    ColumnFamily<String, CompositeColumnName> cf = new ColumnFamily<String, CompositeColumnName>("Order", StringSerializer.get(), CompositeColumnNameSerializer.get());
    IndexAndCf indexAndCf = new IndexAndCf(ClassNameTimeSeriesDBIndex.class, indexCF, keyspace);
    CheckResult checkResult = new CheckResult();
    mockHelper.checkIndexingCF(indexAndCf, false, checkResult);
    assertEquals(0, checkResult.getTotal());
    keyspace.prepareQuery(cf).withCql(String.format("delete from \"Order\" where key='%s'", order.getId())).execute();
    checkResult = new CheckResult();
    mockHelper.checkIndexingCF(indexAndCf, false, checkResult);
    assertEquals(1, checkResult.getTotal());
    keyspace.prepareQuery(indexCF).withCql(mockHelper.getCleanIndexCQL()).execute();
    checkResult = new CheckResult();
    mockHelper.checkIndexingCF(indexAndCf, false, checkResult);
    assertEquals(0, checkResult.getTotal());
}

Example 23 with Keyspace

use of com.netflix.astyanax.Keyspace in project coprhd-controller by CoprHD.

the class DbConsistencyCheckerHelperTest method testCFIndexForOrder.

@Test
public void testCFIndexForOrder() throws Exception {
    DbConsistencyCheckerHelperMock mockHelper = new DbConsistencyCheckerHelperMock((DbClientImpl) getDbClient());
    Order order = new Order();
    order.setId(URIUtil.createId(Order.class));
    order.setLabel("order2");
    order.setSubmittedByUserId("Tom");
    order.setTenant("urn:storageos:TenantOrg:128e0354-c26e-438b-b1e6-1a6ceaa9b380:global");
    order.setIndexed(true);
    getDbClient().createObject(order);
    Keyspace keyspace = ((DbClientImpl) getDbClient()).getLocalContext().getKeyspace();
    ColumnFamily<String, ClassNameTimeSeriesIndexColumnName> userToOrdersByTimeStampCF = new ColumnFamily<String, ClassNameTimeSeriesIndexColumnName>("UserToOrdersByTimeStamp", StringSerializer.get(), ClassNameTimeSeriesSerializer.get());
    ColumnFamily<String, TimeSeriesIndexColumnName> allOrdersByTimeStampCF = new ColumnFamily<String, TimeSeriesIndexColumnName>("AllOrdersByTimeStamp", StringSerializer.get(), TimeSeriesColumnNameSerializer.get());
    ColumnFamily<String, CompositeColumnName> cf = new ColumnFamily<String, CompositeColumnName>("Order", StringSerializer.get(), CompositeColumnNameSerializer.get());
    IndexAndCf indexAndCf = new IndexAndCf(TimeSeriesDbIndex.class, userToOrdersByTimeStampCF, keyspace);
    CheckResult checkResult = new CheckResult();
    mockHelper.checkCFIndices(TypeMap.getDoType(Order.class), true, checkResult);
    assertEquals(0, checkResult.getTotal());
    keyspace.prepareQuery(userToOrdersByTimeStampCF).withCql(String.format("delete from \"UserToOrdersByTimeStamp\" where key='%s'", order.getSubmittedByUserId())).execute();
    keyspace.prepareQuery(userToOrdersByTimeStampCF).withCql(String.format("delete from \"AllOrdersByTimeStamp\" where key='%s'", order.getTenant())).execute();
    checkResult = new CheckResult();
    mockHelper.checkCFIndices(TypeMap.getDoType(Order.class), true, checkResult);
    assertEquals(2, checkResult.getTotal());
}

Example 24 with Keyspace

use of com.netflix.astyanax.Keyspace in project coprhd-controller by CoprHD.

the class InternalDbClientImpl method getLatestModifiedField.

public Column<CompositeColumnName> getLatestModifiedField(DataObjectType type, URI id, Set<String> ignoreList) {
    Column<CompositeColumnName> latestField = null;
    ColumnFamily<String, CompositeColumnName> cf = type.getCF();
    Keyspace ks = this.getKeyspace(type.getDataObjectClass());
    Rows<String, CompositeColumnName> rows = this.queryRowsWithAllColumns(ks, Lists.newArrayList(id), cf);
    if (rows.isEmpty()) {
        log.warn("Can not find the latest modified field of {}", id);
        return latestField;
    }
    long latestTimeStampe = 0;
    for (Column<CompositeColumnName> column : rows.iterator().next().getColumns()) {
        if (ignoreList != null && ignoreList.contains(column.getName().getOne())) {
            continue;
        }
        if (column.getTimestamp() > latestTimeStampe) {
            latestTimeStampe = column.getTimestamp();
            latestField = column;
        }
    }
    return latestField;
}

Example 25 with Keyspace

use of com.netflix.astyanax.Keyspace in project coprhd-controller by CoprHD.

the class DbConsistencyCheckerHelper method checkCFIndices.

/**
 * Scan all the data object records, to find out the object record is existing but the related index is missing.
 *
 * @param doType
 * @param toConsole whether print out in the console
 * @return the number of corrupted data
 * @throws ConnectionException
 */
public void checkCFIndices(DataObjectType doType, boolean toConsole, CheckResult checkResult) throws ConnectionException {
    initSchemaVersions();
    Class objClass = doType.getDataObjectClass();
    if (skipCheckCFs.contains(objClass.getSimpleName())) {
        _log.info("Skip checking CF {}", objClass);
        return;
    } else {
        _log.info("Check Data Object CF {} with double confirmed option: {}", objClass, doubleConfirmed);
    }
    Map<String, ColumnField> indexedFields = new HashMap<String, ColumnField>();
    for (ColumnField field : doType.getColumnFields()) {
        if (field.getIndex() != null) {
            indexedFields.put(field.getName(), field);
        }
    }
    if (indexedFields.isEmpty()) {
        return;
    }
    Keyspace keyspace = dbClient.getKeyspace(objClass);
    ColumnFamilyQuery<String, CompositeColumnName> query = keyspace.prepareQuery(doType.getCF());
    OperationResult<Rows<String, CompositeColumnName>> result = query.getAllRows().setRowLimit(dbClient.DEFAULT_PAGE_SIZE).execute();
    int scannedRows = 0;
    long beginTime = System.currentTimeMillis();
    for (Row<String, CompositeColumnName> objRow : result.getResult()) {
        try {
            boolean inactiveObject = false;
            boolean hasStaleEntries = false;
            scannedRows++;
            Map<String, Column<CompositeColumnName>> distinctColumns = new HashMap<String, Column<CompositeColumnName>>();
            for (Column<CompositeColumnName> column : objRow.getColumns()) {
                distinctColumns.put(column.getName().getOne(), column);
            }
            hasStaleEntries = checkForStaleEntries(distinctColumns);
            if (hasStaleEntries) {
                _log.warn("Data object with key {} has stale entries, don't rebuild index for it.", objRow.getKey());
                continue;
            }
            for (Column<CompositeColumnName> column : objRow.getColumns()) {
                if (column.getName().getOne().equals(DataObject.INACTIVE_FIELD_NAME)) {
                    inactiveObject = column.getBooleanValue();
                    break;
                }
            }
            if (inactiveObject) {
                continue;
            }
            for (Column<CompositeColumnName> column : objRow.getColumns()) {
                if (!indexedFields.containsKey(column.getName().getOne())) {
                    continue;
                }
                // we don't build index if the value is null, refer to ColumnField.
                if (!column.hasValue()) {
                    continue;
                }
                ColumnField indexedField = indexedFields.get(column.getName().getOne());
                String indexKey = getIndexKey(indexedField, column, objRow);
                if (indexKey == null) {
                    continue;
                }
                boolean isColumnInIndex = isColumnInIndex(keyspace, indexedField.getIndexCF(), indexKey, getIndexColumns(indexedField, column, objRow.getKey()));
                if (!isColumnInIndex) {
                    if (doubleConfirmed && isDataObjectRemoved(doType.getDataObjectClass(), objRow.getKey())) {
                        continue;
                    }
                    String dbVersion = findDataCreatedInWhichDBVersion(column.getName().getTimeUUID());
                    checkResult.increaseByVersion(dbVersion);
                    logMessage(String.format("Inconsistency found Object(%s, id: %s, field: %s) is existing, but the related Index(%s, type: %s, id: %s) is missing. This entry is updated by version %s", indexedField.getDataObjectType().getSimpleName(), objRow.getKey(), indexedField.getName(), indexedField.getIndexCF().getName(), indexedField.getIndex().getClass().getSimpleName(), indexKey, dbVersion), true, toConsole);
                    DbCheckerFileWriter.writeTo(DbCheckerFileWriter.WRITER_REBUILD_INDEX, String.format("id:%s, cfName:%s", objRow.getKey(), doType.getCF().getName()));
                }
            }
            if (scannedRows >= THRESHHOLD_FOR_OUTPUT_DEBUG) {
                _log.info("{} data objects have been check with time {}", scannedRows, DurationFormatUtils.formatDurationHMS(System.currentTimeMillis() - beginTime));
                scannedRows = 0;
                beginTime = System.currentTimeMillis();
            }
        } catch (Exception e) {
            _log.warn("exception occurs when checking CF indexes", e);
        }
    }
}