Examples with CloseableIterator org.locationtech.geowave.core.store.CloseableIterator used on opensource projects

Example 1 with CloseableIterator

use of org.locationtech.geowave.core.store.CloseableIterator in project geowave by locationtech.

the class GeoWaveVisibilityIT method testQuery.

public static void testQuery(final DataStore store, final DataStatisticsStore statsStore, final short internalAdapterId, final String[] auths, final boolean spatial, final int expectedResultCount, final int expectedNonNullFieldCount) {
    try (CloseableIterator<SimpleFeature> it = (CloseableIterator) store.query(QueryBuilder.newBuilder().setAuthorizations(auths).constraints(spatial ? new ExplicitSpatialQuery(new GeometryFactory().toGeometry(new Envelope(-1, 1, -1, 1))) : null).build())) {
        int resultCount = 0;
        int nonNullFieldsCount = 0;
        while (it.hasNext()) {
            final SimpleFeature feature = it.next();
            for (int a = 0; a < feature.getAttributeCount(); a++) {
                if (feature.getAttribute(a) != null) {
                    nonNullFieldsCount++;
                }
            }
            resultCount++;
        }
        Assert.assertEquals("Unexpected result count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedResultCount, resultCount);
        Assert.assertEquals("Unexpected non-null field count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedNonNullFieldCount, nonNullFieldsCount);
    }
    final Long count = (Long) store.aggregate(AggregationQueryBuilder.newBuilder().count(getType().getTypeName()).setAuthorizations(auths).constraints(spatial ? new ExplicitSpatialQuery(new GeometryFactory().toGeometry(new Envelope(-1, 1, -1, 1))) : null).build());
    Assert.assertEquals("Unexpected aggregation result count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedResultCount, count.intValue());
    final CountValue countStat = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(getType().getTypeName()).authorizations(auths).build());
    assertNotNull(countStat);
    Assert.assertEquals("Unexpected stats result count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedResultCount, countStat.getValue().intValue());
}

Example 2 with CloseableIterator

use of org.locationtech.geowave.core.store.CloseableIterator in project geowave by locationtech.

the class AttributesSubsetQueryIT method testClientSideFiltering.

@Test
public void testClientSideFiltering() throws IOException {
    final List<String> attributesSubset = Arrays.asList(CITY_ATTRIBUTE, POPULATION_ATTRIBUTE);
    final CloseableIterator<SimpleFeature> results = (CloseableIterator) dataStore.createDataStore().query(QueryBuilder.newBuilder().addTypeName(dataAdapter.getTypeName()).indexName(TestUtils.DEFAULT_SPATIAL_INDEX.getName()).constraints(spatialQuery).build());
    // query expects to match 3 cities from Texas, which should each contain
    // non-null values for a subset of attributes (city, population) and
    // nulls for the rest
    verifyResults(// performs filtering client side
    new FeatureTranslatingIterator(simpleFeatureType, attributesSubset, results), 3, attributesSubset);
}

Example 3 with CloseableIterator

use of org.locationtech.geowave.core.store.CloseableIterator in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}

Example 4 with CloseableIterator

use of org.locationtech.geowave.core.store.CloseableIterator in project geowave by locationtech.

the class GeoWaveMultiProcessIngestIT method testMultiProcessIngest.

@Test
public void testMultiProcessIngest() throws Exception {
    for (int j = 0; j < 10; j++) {
        final Class<?> clazz = GeoWaveMultiProcessIngestIT.class;
        final String javaHome = System.getProperty("java.home");
        final String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
        final String className = clazz.getName();
        final String jarFile = ClasspathUtils.setupPathingJarClassPath(TEMP_DIR, clazz);
        final Index idx1 = SimpleIngest.createSpatialIndex();
        final Index idx2 = SimpleIngest.createSpatialTemporalIndex();
        final DataStore store = dataStorePluginOptions.createDataStore();
        store.addIndex(idx1);
        store.addIndex(idx2);
        final StringBuilder indexNames = new StringBuilder();
        indexNames.append(idx1.getName()).append(",").append(idx2.getName());
        final Configuration conf = new Configuration();
        conf.set(AbstractMapReduceIngest.INDEX_NAMES_KEY, indexNames.toString());
        for (final MetadataType type : MetadataType.values()) {
            // stats and index metadata writers are created elsewhere
            if (!MetadataType.INDEX.equals(type) && !MetadataType.STATISTIC_VALUES.equals(type)) {
                dataStorePluginOptions.createDataStoreOperations().createMetadataWriter(type).close();
            }
        }
        GeoWaveOutputFormat.addIndex(conf, idx1);
        GeoWaveOutputFormat.addIndex(conf, idx2);
        GeoWaveOutputFormat.setStoreOptions(conf, dataStorePluginOptions);
        Assert.assertTrue(TEMP_DIR.exists() || TEMP_DIR.mkdirs());
        final File configFile = new File(TEMP_DIR, "hadoop-job.conf");
        Assert.assertTrue(!configFile.exists() || configFile.delete());
        Assert.assertTrue(configFile.createNewFile());
        try (DataOutputStream dataOut = new DataOutputStream(new FileOutputStream(configFile))) {
            conf.write(dataOut);
        }
        final List<ProcessBuilder> bldrs = new ArrayList<>();
        for (int i = 0; i < NUM_PROCESSES; i++) {
            final ArrayList<String> argList = new ArrayList<>();
            argList.addAll(Arrays.asList(javaBin, "-cp", jarFile, className, new Integer(i * 10000).toString()));
            final ProcessBuilder builder = new ProcessBuilder(argList);
            builder.directory(TEMP_DIR);
            builder.inheritIO();
            bldrs.add(builder);
        }
        final List<Process> processes = bldrs.stream().map(b -> {
            try {
                return b.start();
            } catch (final IOException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
            }
            return null;
        }).collect(Collectors.toList());
        Assert.assertFalse(processes.stream().anyMatch(Objects::isNull));
        processes.forEach(p -> {
            try {
                p.waitFor();
            } catch (final InterruptedException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        });
        try (CloseableIterator<Object> it = store.query(QueryBuilder.newBuilder().build())) {
            Assert.assertEquals(2701 * NUM_PROCESSES, Iterators.size(it));
        }
        try (CloseableIterator<SimpleFeature> it = store.query(VectorQueryBuilder.newBuilder().indexName(idx1.getName()).build())) {
            Assert.assertEquals(2701 * NUM_PROCESSES, Iterators.size(it));
        }
        try (CloseableIterator<SimpleFeature> it = store.query(VectorQueryBuilder.newBuilder().indexName(idx2.getName()).build())) {
            Assert.assertEquals(2701 * NUM_PROCESSES, Iterators.size(it));
        }
        try (CloseableIterator<SimpleFeature> it = store.query(VectorQueryBuilder.newBuilder().constraints(VectorQueryBuilder.newBuilder().constraintsFactory().spatialTemporalConstraints().spatialConstraints(GeometryUtils.GEOMETRY_FACTORY.toGeometry(new Envelope(-172, 172, -82, 82))).build()).build())) {
            Assert.assertEquals(2277 * NUM_PROCESSES, Iterators.size(it));
        }
        final long epochTime = 1609459200000L;
        final long startTime = epochTime + TimeUnit.DAYS.toMillis(15);
        final long endTime = epochTime + TimeUnit.DAYS.toMillis(345);
        try (CloseableIterator<SimpleFeature> it = store.query(VectorQueryBuilder.newBuilder().constraints(VectorQueryBuilder.newBuilder().constraintsFactory().spatialTemporalConstraints().spatialConstraints(GeometryUtils.GEOMETRY_FACTORY.toGeometry(new Envelope(-172, 172, -82, 82))).addTimeRange(new Date(startTime), new Date(endTime)).build()).build())) {
            Assert.assertEquals(2178 * NUM_PROCESSES, Iterators.size(it));
        }
        TestUtils.deleteAll(getDataStorePluginOptions());
    }
}

Example 5 with CloseableIterator

use of org.locationtech.geowave.core.store.CloseableIterator in project geowave by locationtech.

the class CentroidManagerGeoWave method getRawCentroids.

@SuppressWarnings("unchecked")
protected CloseableIterator<T> getRawCentroids(final String batchId, final String groupID) throws IOException {
    final FilterFactoryImpl factory = new FilterFactoryImpl();
    final Expression expB1 = factory.property(ClusterFeatureAttribute.BATCH_ID.attrName());
    final Expression expB2 = factory.literal(batchId);
    final Filter batchIdFilter = factory.equal(expB1, expB2, false);
    Filter finalFilter = batchIdFilter;
    if (groupID != null) {
        final Expression exp1 = factory.property(ClusterFeatureAttribute.GROUP_ID.attrName());
        final Expression exp2 = factory.literal(groupID);
        // ignore levels for group IDS
        finalFilter = factory.and(factory.equal(exp1, exp2, false), batchIdFilter);
    } else if (level > 0) {
        final Expression exp1 = factory.property(ClusterFeatureAttribute.ZOOM_LEVEL.attrName());
        final Expression exp2 = factory.literal(level);
        finalFilter = factory.and(factory.equal(exp1, exp2, false), batchIdFilter);
    }
    final VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName());
    return (CloseableIterator<T>) dataStore.query(bldr.constraints(bldr.constraintsFactory().filterConstraints(finalFilter)).build());
}