Examples with CommonIndexModel org.locationtech.geowave.core.store.index.CommonIndexModel used on opensource projects

Example 1 with CommonIndexModel

use of org.locationtech.geowave.core.store.index.CommonIndexModel in project geowave by locationtech.

the class AbstractPartitioner method initialize.

public void initialize(final ScopedJobConfiguration config) throws IOException {
    distancePerDimension = getDistances(config);
    this.precisionFactor = config.getDouble(Partition.PARTITION_PRECISION, 1.0);
    if ((precisionFactor < 0) || (precisionFactor > 1.0)) {
        throw new IllegalArgumentException(String.format("Precision value must be between 0 and 1: %.6f", precisionFactor));
    }
    try {
        final IndexModelBuilder builder = config.getInstance(CommonParameters.Common.INDEX_MODEL_BUILDER_CLASS, IndexModelBuilder.class, SpatialIndexModelBuilder.class);
        final CommonIndexModel model = builder.buildModel();
        if (model.getDimensions().length > distancePerDimension.length) {
            final double[] newDistancePerDimension = new double[model.getDimensions().length];
            for (int j = 0; j < newDistancePerDimension.length; j++) {
                newDistancePerDimension[j] = distancePerDimension[j < distancePerDimension.length ? j : (distancePerDimension.length - 1)];
            }
            distancePerDimension = newDistancePerDimension;
        }
        this.initIndex(model, distancePerDimension);
    } catch (InstantiationException | IllegalAccessException e) {
        throw new IOException(e);
    }
}

Example 2 with CommonIndexModel

use of org.locationtech.geowave.core.store.index.CommonIndexModel in project geowave by locationtech.

the class FilterConstraints method getIndexData.

/**
 * Get the multi-dimensional index data from these constraints.
 *
 * @return the multi-dimensional index data
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public List<MultiDimensionalIndexData<V>> getIndexData() {
    if (cachedIndexData == null) {
        if ((adapter == null) || (index == null) || (indexMapping == null)) {
            return Lists.newArrayList();
        }
        if (index instanceof CustomIndex) {
            final TextIndexStrategy indexStrategy = (TextIndexStrategy) ((CustomIndex) index).getCustomIndexStrategy();
            if (!(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter)) {
                throw new RuntimeException("Unable to determine adapter field used by text index.");
            }
            final String fieldName = ((AdapterFieldTextIndexEntryConverter) indexStrategy.getEntryConverter()).getFieldName();
            final IndexFieldConstraints<?> fieldConstraint = fieldConstraints.get(fieldName);
            final List<DimensionConstraints<String>> dimensionConstraints = Lists.newArrayList();
            if (fieldConstraint == null) {
                dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((String) null, (String) null, true, true, true))));
            } else if (fieldConstraint instanceof TextFieldConstraints) {
                final DimensionConstraints<String> dimensionConstraint = ((TextFieldConstraints) fieldConstraint).getDimensionRanges(0);
                if (dimensionConstraint == null) {
                    dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((String) null, (String) null, true, true, true))));
                } else {
                    dimensionConstraints.add(dimensionConstraint);
                }
            } else {
                throw new RuntimeException("Non-text field constraints cannot be used for a text index.");
            }
            cachedIndexData = (List) TextFieldConstraints.toIndexData(dimensionConstraints);
        } else {
            // Right now all index strategies that aren't custom are numeric
            final CommonIndexModel indexModel = index.getIndexModel();
            final int numStrategyDimensions = index.getIndexStrategy().getOrderedDimensionDefinitions().length;
            final List<DimensionConstraints<Double>> dimensionConstraints = Lists.newArrayListWithCapacity(numStrategyDimensions);
            final Map<String, Integer> indexFieldDimensions = Maps.newHashMap();
            final NumericDimensionField<?>[] dimensions = indexModel.getDimensions();
            int dimensionIndex = 0;
            for (final NumericDimensionField<?> indexField : dimensions) {
                if (dimensionIndex >= numStrategyDimensions) {
                    // Only build constraints for dimensions used by the index strategy.
                    break;
                }
                dimensionIndex++;
                final String indexFieldName = indexField.getFieldName();
                if (!indexFieldDimensions.containsKey(indexFieldName)) {
                    indexFieldDimensions.put(indexFieldName, 0);
                }
                final int indexFieldDimension = indexFieldDimensions.get(indexFieldName);
                final IndexFieldMapper<?, ?> mapper = indexMapping.getMapperForIndexField(indexFieldName);
                final String[] adapterFields = mapper.getIndexOrderedAdapterFields();
                IndexFieldConstraints<?> fieldConstraint = null;
                if (adapterFields.length > 1 && isSingleDimension(indexFieldName, dimensions)) {
                    // constraints
                    for (int i = 0; i < adapterFields.length; i++) {
                        final IndexFieldConstraints<?> constraint = fieldConstraints.get(adapterFields[i]);
                        if (fieldConstraint == null) {
                            fieldConstraint = constraint;
                        } else {
                            fieldConstraint.and((IndexFieldConstraints) constraint);
                        }
                    }
                } else {
                    fieldConstraint = fieldConstraints.get(adapterFields[indexFieldDimension % adapterFields.length]);
                }
                if (fieldConstraint == null) {
                    dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((Double) null, (Double) null, true, true, true))));
                } else if (fieldConstraint instanceof NumericFieldConstraints) {
                    final DimensionConstraints<Double> dimensionConstraint = ((NumericFieldConstraints) fieldConstraint).getDimensionRanges(indexFieldDimension % fieldConstraint.getDimensionCount());
                    if (dimensionConstraint == null) {
                        dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((Double) null, (Double) null, true, true, true))));
                    } else {
                        dimensionConstraints.add(dimensionConstraint);
                    }
                    indexFieldDimensions.put(indexFieldName, indexFieldDimension + 1);
                } else {
                    throw new RuntimeException("Non-numeric field constraints cannot be used for a numeric index.");
                }
            }
            cachedIndexData = (List) NumericFieldConstraints.toIndexData(dimensionConstraints);
        }
    }
    return cachedIndexData;
}

Example 3 with CommonIndexModel

use of org.locationtech.geowave.core.store.index.CommonIndexModel in project geowave by locationtech.

the class BaseDataStoreUtils method getWriteInfo.

protected static <T> IntermediaryWriteEntryInfo getWriteInfo(final T entry, final InternalDataAdapter<T> adapter, final AdapterToIndexMapping indexMapping, final Index index, final VisibilityHandler visibilityHandler, final boolean secondaryIndex, final boolean dataIdIndex, final boolean visibilityEnabled) {
    final CommonIndexModel indexModel = index.getIndexModel();
    final short internalAdapterId = adapter.getAdapterId();
    final byte[] dataId = adapter.getDataId(entry);
    final AdapterPersistenceEncoding encodedData = adapter.encode(entry, indexMapping, index);
    if (encodedData == null) {
        // The entry could not be encoded to the index, but this could be due to a null value in one
        // of the index fields, which is possible in attribute indices
        LOGGER.info("Indexing failed to produce insertion ids; entry [" + StringUtils.stringFromBinary(adapter.getDataId(entry)) + "] not saved for index '" + index.getName() + "'.");
        return new IntermediaryWriteEntryInfo(dataId, internalAdapterId, new InsertionIds(), new GeoWaveValueImpl[0]);
    }
    final InsertionIds insertionIds;
    if (index instanceof CustomIndexStrategy) {
        insertionIds = ((CustomIndexStrategy) index).getInsertionIds(entry);
    } else {
        insertionIds = dataIdIndex ? null : encodedData.getInsertionIds(index);
    }
    if (dataIdIndex) {
        return getWriteInfoDataIDIndex(entry, dataId, encodedData, adapter, indexMapping, index, visibilityHandler, visibilityEnabled);
    }
    if (insertionIds.isEmpty()) {
        // we can allow some entries to not be indexed within every index for flexibility, and
        // therefore this should just be info level
        LOGGER.info("Indexing failed to produce insertion ids; entry [" + StringUtils.stringFromBinary(adapter.getDataId(entry)) + "] not saved for index '" + index.getName() + "'.");
        return new IntermediaryWriteEntryInfo(dataId, internalAdapterId, insertionIds, new GeoWaveValueImpl[0]);
    }
    final VisibilityComposer commonIndexVisibility = new VisibilityComposer();
    if (visibilityEnabled && (visibilityHandler != null)) {
        for (final Entry<String, Object> fieldValue : encodedData.getCommonData().getValues().entrySet()) {
            addIndexFieldVisibility(entry, adapter, indexMapping, visibilityHandler, fieldValue.getKey(), commonIndexVisibility);
        }
    }
    if (secondaryIndex && DataIndexUtils.adapterSupportsDataIndex(adapter)) {
        return new IntermediaryWriteEntryInfo(dataId, internalAdapterId, insertionIds, new GeoWaveValue[] { new GeoWaveValueImpl(new byte[0], StringUtils.stringToBinary(commonIndexVisibility.composeVisibility()), new byte[0]) });
    }
    final List<FieldInfo<?>> fieldInfoList = new ArrayList<>();
    addCommonFields(adapter, indexMapping, entry, index, indexModel, visibilityHandler, encodedData, visibilityEnabled, fieldInfoList);
    for (final Entry<String, Object> fieldValue : encodedData.getAdapterExtendedData().getValues().entrySet()) {
        if (fieldValue.getValue() != null) {
            final FieldInfo<?> fieldInfo = getFieldInfo(adapter, adapter, indexMapping, fieldValue.getKey(), fieldValue.getValue(), entry, visibilityHandler, visibilityEnabled, false);
            if (fieldInfo != null) {
                fieldInfoList.add(fieldInfo);
            }
        }
    }
    return new IntermediaryWriteEntryInfo(dataId, internalAdapterId, insertionIds, BaseDataStoreUtils.composeFlattenedFields(fieldInfoList, indexModel, adapter, commonIndexVisibility, dataIdIndex));
}

Example 4 with CommonIndexModel

use of org.locationtech.geowave.core.store.index.CommonIndexModel in project geowave by locationtech.

the class BasicQueryByClassTest method testDisjointCasesWithPersistence.

@Test
public void testDisjointCasesWithPersistence() {
    final List<MultiDimensionalNumericData> expectedResults = new ArrayList<>();
    expectedResults.add(new BasicNumericDataset(new NumericData[] { new ConstrainedIndexValue(0.3, 0.7), new ConstrainedIndexValue(0.1, 2.3) }));
    expectedResults.add(new BasicNumericDataset(new NumericData[] { new ConstrainedIndexValue(0.3, 0.7), new ConstrainedIndexValue(3.4, 3.7) }));
    final ConstraintSet cs1 = new ConstraintSet();
    cs1.addConstraint(ExampleDimensionOne.class, new ConstraintData(new ConstrainedIndexValue(0.3, 0.5), true));
    cs1.addConstraint(ExampleDimensionOne.class, new ConstraintData(new ConstrainedIndexValue(0.4, 0.7), true));
    final ConstraintSet cs2a = new ConstraintSet();
    cs2a.addConstraint(ExampleDimensionTwo.class, new ConstraintData(new ConstrainedIndexValue(0.1, 0.2), true));
    cs2a.addConstraint(ExampleDimensionTwo.class, new ConstraintData(new ConstrainedIndexValue(2.1, 2.3), true));
    final ConstraintSet cs2b = new ConstraintSet();
    cs2b.addConstraint(ExampleDimensionTwo.class, new ConstraintData(new ConstrainedIndexValue(3.4, 3.7), true));
    final ConstraintsByClass constraints = new ConstraintsByClass(Arrays.asList(cs2a, cs2b)).merge(Collections.singletonList(cs1));
    assertEquals(expectedResults, constraints.getIndexConstraints(new IndexImpl(new ExampleNumericIndexStrategy(), null)));
    final byte[] image = new BasicQueryByClass(constraints).toBinary();
    final BasicQueryByClass query = new BasicQueryByClass();
    query.fromBinary(image);
    final Index index = new CustomNameIndex(new ExampleNumericIndexStrategy(), new BasicIndexModel(new NumericDimensionField[] { new ExampleDimensionOne(), new ExampleDimensionTwo() }), "22");
    assertEquals(expectedResults, query.getIndexConstraints(index));
    final List<QueryFilter> filters = query.createFilters(index);
    assertEquals(1, filters.size());
    final Map<String, ConstrainedIndexValue> fieldIdToValueMap = new HashMap<>();
    fieldIdToValueMap.put("one", new ConstrainedIndexValue(0.4, 0.4));
    fieldIdToValueMap.put("two", new ConstrainedIndexValue(0.5, 0.5));
    final CommonIndexModel model = null;
    assertTrue(filters.get(0).accept(model, new CommonIndexedPersistenceEncoding((short) 1, StringUtils.stringToBinary("data"), StringUtils.stringToBinary("partition"), StringUtils.stringToBinary("sort"), // duplicate count
    1, new MultiFieldPersistentDataset(fieldIdToValueMap), null)));
    fieldIdToValueMap.put("one", new ConstrainedIndexValue(0.1, 0.1));
    assertFalse(filters.get(0).accept(model, new CommonIndexedPersistenceEncoding((short) 1, StringUtils.stringToBinary("data"), StringUtils.stringToBinary("partition"), StringUtils.stringToBinary("sort"), // duplicate count
    1, new MultiFieldPersistentDataset(fieldIdToValueMap), null)));
    fieldIdToValueMap.put("one", new ConstrainedIndexValue(0.4, 0.4));
    fieldIdToValueMap.put("two", new ConstrainedIndexValue(5.0, 5.0));
    assertFalse(filters.get(0).accept(model, new CommonIndexedPersistenceEncoding((short) 1, StringUtils.stringToBinary("data"), StringUtils.stringToBinary("partition"), StringUtils.stringToBinary("sort"), // duplicate count
    1, new MultiFieldPersistentDataset(fieldIdToValueMap), null)));
    /**
     * Tests the 'OR' Case
     */
    fieldIdToValueMap.put("two", new ConstrainedIndexValue(3.5, 3.5));
    assertTrue(filters.get(0).accept(model, new CommonIndexedPersistenceEncoding((short) 1, StringUtils.stringToBinary("data"), StringUtils.stringToBinary("partition"), StringUtils.stringToBinary("sort"), // duplicate count
    1, new MultiFieldPersistentDataset(fieldIdToValueMap), null)));
}

Example 5 with CommonIndexModel

use of org.locationtech.geowave.core.store.index.CommonIndexModel in project geowave by locationtech.

the class MigrationCommand method migrate0to1.

public void migrate0to1(final DataStorePluginOptions options, final DataStoreOperations operations, final Console console) {
    console.println("Migration 1.x -> 2.x");
    final DataStore dataStore = options.createDataStore();
    console.println("  Migrating data type adapters...");
    final PersistentAdapterStore adapterStore = options.createAdapterStore();
    final List<Short> adapterIDs = Lists.newArrayList();
    int migratedAdapters = 0;
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> adapter : adapters) {
        adapterIDs.add(adapter.getAdapterId());
        if (adapter instanceof LegacyInternalDataAdapterWrapper) {
            adapterStore.removeAdapter(adapter.getAdapterId());
            // Write updated adapter
            adapterStore.addAdapter(((LegacyInternalDataAdapterWrapper<?>) adapter).getUpdatedAdapter());
            migratedAdapters++;
        } else if (adapter.getAdapter() instanceof LegacyFeatureDataAdapter) {
            final FeatureDataAdapter updatedAdapter = ((LegacyFeatureDataAdapter) adapter.getAdapter()).getUpdatedAdapter();
            final VisibilityHandler visibilityHandler = ((LegacyFeatureDataAdapter) adapter.getAdapter()).getVisibilityHandler();
            // Write updated adapter
            adapterStore.removeAdapter(adapter.getAdapterId());
            adapterStore.addAdapter(updatedAdapter.asInternalAdapter(adapter.getAdapterId(), visibilityHandler));
            migratedAdapters++;
        }
    }
    if (migratedAdapters > 0) {
        console.println("    Migrated " + migratedAdapters + " data type adapters.");
    } else {
        console.println("    No data type adapters needed to be migrated.");
    }
    console.println("  Migrating indices...");
    final IndexStore indexStore = options.createIndexStore();
    int migratedIndices = 0;
    try (CloseableIterator<Index> indices = indexStore.getIndices()) {
        while (indices.hasNext()) {
            final Index index = indices.next();
            final CommonIndexModel indexModel = index.getIndexModel();
            // if the index model uses any spatial fields, update and re-write
            if ((indexModel != null) && (indexModel instanceof BasicIndexModel)) {
                final NumericDimensionField<?>[] oldFields = indexModel.getDimensions();
                final NumericDimensionField<?>[] updatedFields = new NumericDimensionField<?>[oldFields.length];
                boolean updated = false;
                for (int i = 0; i < oldFields.length; i++) {
                    if (oldFields[i] instanceof LegacySpatialField) {
                        updatedFields[i] = ((LegacySpatialField<?>) oldFields[i]).getUpdatedField(index);
                        updated = true;
                    } else {
                        updatedFields[i] = oldFields[i];
                    }
                }
                if (updated) {
                    ((BasicIndexModel) indexModel).init(updatedFields);
                    indexStore.removeIndex(index.getName());
                    indexStore.addIndex(index);
                    migratedIndices++;
                }
            }
        }
    }
    if (migratedIndices > 0) {
        console.println("    Migrated " + migratedIndices + " indices.");
    } else {
        console.println("    No indices needed to be migrated.");
    }
    console.println("  Migrating index mappings...");
    // Rewrite adapter to index mappings
    final LegacyAdapterIndexMappingStore legacyIndexMappings = new LegacyAdapterIndexMappingStore(operations, options.getFactoryOptions().getStoreOptions());
    final AdapterIndexMappingStore indexMappings = options.createAdapterIndexMappingStore();
    console.println("    Writing new mappings...");
    int indexMappingCount = 0;
    for (final Short adapterId : adapterIDs) {
        final LegacyAdapterToIndexMapping mapping = legacyIndexMappings.getIndicesForAdapter(adapterId);
        final InternalDataAdapter<?> adapter = adapterStore.getAdapter(adapterId);
        for (final String indexName : mapping.getIndexNames()) {
            indexMappings.addAdapterIndexMapping(BaseDataStoreUtils.mapAdapterToIndex(adapter, indexStore.getIndex(indexName)));
            indexMappingCount++;
        }
    }
    if (indexMappingCount > 0) {
        console.println("    Migrated " + indexMappingCount + " index mappings.");
        console.println("    Deleting legacy index mappings...");
        try (MetadataDeleter deleter = operations.createMetadataDeleter(MetadataType.LEGACY_INDEX_MAPPINGS)) {
            deleter.delete(new MetadataQuery(null));
        } catch (final Exception e) {
            LOGGER.warn("Error deleting legacy index mappings", e);
        }
    } else {
        console.println("    No index mappings to migrate.");
    }
    // Update statistics
    console.println("  Migrating statistics...");
    final List<Statistic<?>> defaultStatistics = new ArrayList<>();
    for (final Index index : dataStore.getIndices()) {
        if (index instanceof DefaultStatisticsProvider) {
            defaultStatistics.addAll(((DefaultStatisticsProvider) index).getDefaultStatistics());
        }
    }
    for (final DataTypeAdapter<?> adapter : dataStore.getTypes()) {
        final DefaultStatisticsProvider defaultStatProvider = BaseDataStoreUtils.getDefaultStatisticsProvider(adapter);
        if (defaultStatProvider != null) {
            defaultStatistics.addAll(defaultStatProvider.getDefaultStatistics());
        }
    }
    console.println("    Calculating updated statistics...");
    dataStore.addStatistic(defaultStatistics.toArray(new Statistic[defaultStatistics.size()]));
    console.println("    Deleting legacy statistics...");
    try (MetadataDeleter deleter = operations.createMetadataDeleter(MetadataType.LEGACY_STATISTICS)) {
        deleter.delete(new MetadataQuery(null));
    } catch (final Exception e) {
        LOGGER.warn("Error deleting legacy statistics", e);
    }
}