Search in sources :

Example 1 with RoundRobinKeyIndexStrategy

use of org.locationtech.geowave.core.index.simple.RoundRobinKeyIndexStrategy in project geowave by locationtech.

the class RocksDBLockfileTest method testLockfile.

private void testLockfile(final int numThreads, final boolean secondaryIndexing) {
    final RocksDBOptions options = new RocksDBOptions();
    options.setDirectory(DEFAULT_DB_DIRECTORY);
    options.getStoreOptions().setSecondaryIndexing(secondaryIndexing);
    final DataStore store = new RocksDBStoreFactoryFamily().getDataStoreFactory().createStore(options);
    store.deleteAll();
    store.addType(BasicDataTypeAdapter.newAdapter(POI_TYPE_NAME, POI.class, "name"));
    Index index = AttributeDimensionalityTypeProvider.createIndexFromOptions(store, new AttributeIndexOptions(POI_TYPE_NAME, "latitude"));
    index = new CustomNameIndex(new CompoundIndexStrategy(new RoundRobinKeyIndexStrategy(32), index.getIndexStrategy()), index.getIndexModel(), index.getName() + "_" + PartitionStrategy.ROUND_ROBIN.name() + "_" + 32);
    final Index latAttributeIndex = new IndexWrapper(index);
    store.addIndex(POI_TYPE_NAME, latAttributeIndex);
    final DataStore store2 = new RocksDBStoreFactoryFamily().getDataStoreFactory().createStore(options);
    IntStream.range(0, numThreads).mapToObj(i -> CompletableFuture.runAsync(() -> {
        double offset = i * numThreads;
        try (Writer<POI> w = store.createWriter(POI_TYPE_NAME)) {
            w.write(new POI("name" + offset, offset, offset));
        }
        try (CloseableIterator<POI> poiIt = store2.query(QueryBuilder.newBuilder(POI.class).build())) {
            if (numThreads == 1) {
                Assert.assertEquals(1, Iterators.size(poiIt));
            } else {
                Assert.assertTrue(Iterators.size(poiIt) >= 1);
            }
        }
        offset++;
        try (Writer<POI> w = store2.createWriter(POI_TYPE_NAME)) {
            w.write(new POI("name" + offset, offset, offset));
        }
        try (CloseableIterator<POI> poiIt = store.query(QueryBuilder.newBuilder(POI.class).build())) {
            if (numThreads == 1) {
                Assert.assertEquals(2, Iterators.size(poiIt));
            } else {
                Assert.assertTrue(Iterators.size(poiIt) >= 2);
            }
        }
        offset++;
        try (Writer<POI> w = store2.createWriter(POI_TYPE_NAME)) {
            w.write(new POI("name" + offset, offset, offset));
            try (CloseableIterator<POI> poiIt = store.query(QueryBuilder.newBuilder(POI.class).build())) {
                if (numThreads == 1) {
                    Assert.assertEquals(2, Iterators.size(poiIt));
                } else {
                    Assert.assertTrue(Iterators.size(poiIt) >= 2);
                }
            }
            w.flush();
            try (CloseableIterator<POI> poiIt = store.query(QueryBuilder.newBuilder(POI.class).build())) {
                if (numThreads == 1) {
                    Assert.assertEquals(3, Iterators.size(poiIt));
                } else {
                    Assert.assertTrue(Iterators.size(poiIt) >= 3);
                }
            }
        }
        try (CloseableIterator<POI> poiIt = store2.query(QueryBuilder.newBuilder(POI.class).build())) {
            if (numThreads == 1) {
                Assert.assertEquals(3, Iterators.size(poiIt));
            } else {
                Assert.assertTrue(Iterators.size(poiIt) >= 3);
            }
        }
    }));
    store.deleteAll();
}
Also used : CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) AttributeIndexOptions(org.locationtech.geowave.core.store.index.AttributeIndexOptions) CompoundIndexStrategy(org.locationtech.geowave.core.index.CompoundIndexStrategy) DataStore(org.locationtech.geowave.core.store.api.DataStore) CustomNameIndex(org.locationtech.geowave.core.store.index.CustomNameIndex) Index(org.locationtech.geowave.core.store.api.Index) RoundRobinKeyIndexStrategy(org.locationtech.geowave.core.index.simple.RoundRobinKeyIndexStrategy) RocksDBOptions(org.locationtech.geowave.datastore.rocksdb.config.RocksDBOptions) CustomNameIndex(org.locationtech.geowave.core.store.index.CustomNameIndex) Writer(org.locationtech.geowave.core.store.api.Writer)

Example 2 with RoundRobinKeyIndexStrategy

use of org.locationtech.geowave.core.index.simple.RoundRobinKeyIndexStrategy in project geowave by locationtech.

the class CompoundHierarchicalIndexStrategyWrapper method findHierarchicalStrategy.

public static HierarchicalNumericIndexStrategy findHierarchicalStrategy(final NumericIndexStrategy indexStrategy, final List<CompoundIndexStrategy> parentStrategies) {
    if (indexStrategy instanceof HierarchicalNumericIndexStrategy) {
        return (HierarchicalNumericIndexStrategy) indexStrategy;
    }
    if (indexStrategy instanceof CompoundIndexStrategy) {
        final PartitionIndexStrategy<MultiDimensionalNumericData, MultiDimensionalNumericData> primaryIndex = ((CompoundIndexStrategy) indexStrategy).getPrimarySubStrategy();
        final NumericIndexStrategy secondaryIndex = ((CompoundIndexStrategy) indexStrategy).getSecondarySubStrategy();
        // warn if round robin is used
        if (primaryIndex instanceof RoundRobinKeyIndexStrategy) {
            LOGGER.warn("Round Robin partitioning won't work correctly with raster merge strategies");
        } else if (secondaryIndex instanceof RoundRobinKeyIndexStrategy) {
            LOGGER.warn("Round Robin partitioning won't work correctly with raster merge strategies");
        }
        final HierarchicalNumericIndexStrategy secondary = findHierarchicalStrategy(secondaryIndex);
        if (secondary != null) {
            // add it to beginning because we are recursing back from the
            // leaf strategy up to the parent
            parentStrategies.add(0, (CompoundIndexStrategy) indexStrategy);
            return secondary;
        }
    }
    return null;
}
Also used : MultiDimensionalNumericData(org.locationtech.geowave.core.index.numeric.MultiDimensionalNumericData) CompoundIndexStrategy(org.locationtech.geowave.core.index.CompoundIndexStrategy) HierarchicalNumericIndexStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy) RoundRobinKeyIndexStrategy(org.locationtech.geowave.core.index.simple.RoundRobinKeyIndexStrategy) HierarchicalNumericIndexStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy) NumericIndexStrategy(org.locationtech.geowave.core.index.NumericIndexStrategy)

Example 3 with RoundRobinKeyIndexStrategy

use of org.locationtech.geowave.core.index.simple.RoundRobinKeyIndexStrategy in project geowave by locationtech.

the class AccumuloUtils method setSplitsByRandomPartitions.

/**
 * Set splits on a table based on a partition ID
 */
public static void setSplitsByRandomPartitions(final Connector connector, final String namespace, final Index index, final int randomPartitions) throws AccumuloException, AccumuloSecurityException, IOException, TableNotFoundException {
    final AccumuloOperations operations = new AccumuloOperations(connector, namespace, new AccumuloOptions());
    final RoundRobinKeyIndexStrategy partitions = new RoundRobinKeyIndexStrategy(randomPartitions);
    operations.createTable(index.getName(), true, true);
    for (final byte[] p : partitions.getPartitionKeys()) {
        operations.ensurePartition(new ByteArray(p), index.getName());
    }
}
Also used : AccumuloOperations(org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations) AccumuloOptions(org.locationtech.geowave.datastore.accumulo.config.AccumuloOptions) ByteArray(org.locationtech.geowave.core.index.ByteArray) RoundRobinKeyIndexStrategy(org.locationtech.geowave.core.index.simple.RoundRobinKeyIndexStrategy)

Aggregations

RoundRobinKeyIndexStrategy (org.locationtech.geowave.core.index.simple.RoundRobinKeyIndexStrategy)3 CompoundIndexStrategy (org.locationtech.geowave.core.index.CompoundIndexStrategy)2 ByteArray (org.locationtech.geowave.core.index.ByteArray)1 HierarchicalNumericIndexStrategy (org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy)1 NumericIndexStrategy (org.locationtech.geowave.core.index.NumericIndexStrategy)1 MultiDimensionalNumericData (org.locationtech.geowave.core.index.numeric.MultiDimensionalNumericData)1 CloseableIterator (org.locationtech.geowave.core.store.CloseableIterator)1 DataStore (org.locationtech.geowave.core.store.api.DataStore)1 Index (org.locationtech.geowave.core.store.api.Index)1 Writer (org.locationtech.geowave.core.store.api.Writer)1 AttributeIndexOptions (org.locationtech.geowave.core.store.index.AttributeIndexOptions)1 CustomNameIndex (org.locationtech.geowave.core.store.index.CustomNameIndex)1 AccumuloOptions (org.locationtech.geowave.datastore.accumulo.config.AccumuloOptions)1 AccumuloOperations (org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations)1 RocksDBOptions (org.locationtech.geowave.datastore.rocksdb.config.RocksDBOptions)1