Examples with ExpectedResults org.locationtech.geowave.test.TestUtils.ExpectedResults used on opensource projects

Example 1 with ExpectedResults

use of org.locationtech.geowave.test.TestUtils.ExpectedResults in project geowave by locationtech.

the class BasicMapReduceIT method testIngestOsmGpxMultipleIndices.

@Test
public void testIngestOsmGpxMultipleIndices() throws Exception {
    TestUtils.deleteAll(dataStorePluginOptions);
    // ingest the data set into multiple indices and then try several query
    // methods, by adapter and by index
    MapReduceTestUtils.testMapReduceIngest(dataStorePluginOptions, DimensionalityType.SPATIAL_AND_SPATIAL_TEMPORAL, OSM_GPX_INPUT_DIR);
    final DataTypeAdapter<SimpleFeature>[] adapters = new GpxIngestPlugin().getDataAdapters(null);
    final org.locationtech.geowave.core.store.api.DataStore geowaveStore = dataStorePluginOptions.createDataStore();
    final Map<String, ExpectedResults> adapterIdToResultsMap = new HashMap<>();
    for (final DataTypeAdapter<SimpleFeature> adapter : adapters) {
        adapterIdToResultsMap.put(adapter.getTypeName(), TestUtils.getExpectedResults(geowaveStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).build())));
    }
    final List<DataTypeAdapter<?>> firstTwoAdapters = new ArrayList<>();
    firstTwoAdapters.add(adapters[0]);
    firstTwoAdapters.add(adapters[1]);
    final ExpectedResults firstTwoAdaptersResults = TestUtils.getExpectedResults(geowaveStore.query(QueryBuilder.newBuilder().addTypeName(adapters[0].getTypeName()).addTypeName(adapters[1].getTypeName()).build()));
    final ExpectedResults fullDataSetResults = TestUtils.getExpectedResults(geowaveStore.query(QueryBuilder.newBuilder().build()));
    // just for sanity verify its greater than 0 (ie. that data was actually
    // ingested in the first place)
    Assert.assertTrue("There is no data ingested from OSM GPX test files", fullDataSetResults.count > 0);
    // now that we have expected results, run map-reduce export and
    // re-ingest it
    testMapReduceExportAndReingest(DimensionalityType.SPATIAL_AND_SPATIAL_TEMPORAL);
    // first try each adapter individually
    for (final DataTypeAdapter<SimpleFeature> adapter : adapters) {
        final ExpectedResults expResults = adapterIdToResultsMap.get(adapter.getTypeName());
        if (expResults.count > 0) {
            LOGGER.error("Running test for adapter " + adapter.getTypeName());
            runTestJob(expResults, null, new DataTypeAdapter[] { adapter }, null);
        }
    }
    // then try the first 2 adapters, and may as well try with both indices
    // set (should be the default behavior anyways)
    runTestJob(firstTwoAdaptersResults, null, new DataTypeAdapter[] { adapters[0], adapters[1] }, null);
    // now try all adapters and the spatial temporal index, the result
    // should be the full data set
    runTestJob(fullDataSetResults, null, adapters, TestUtils.DEFAULT_SPATIAL_TEMPORAL_INDEX);
    // and finally run with nothing set, should be the full data set
    runTestJob(fullDataSetResults, null, null, null);
}

Example 2 with ExpectedResults

use of org.locationtech.geowave.test.TestUtils.ExpectedResults in project geowave by locationtech.

the class BasicMapReduceIT method testIngestAndQueryGeneralGpx.

@Test
public void testIngestAndQueryGeneralGpx() throws Exception {
    TestUtils.deleteAll(dataStorePluginOptions);
    MapReduceTestUtils.testMapReduceIngest(dataStorePluginOptions, DimensionalityType.SPATIAL, GENERAL_GPX_INPUT_GPX_DIR);
    final File gpxInputDir = new File(GENERAL_GPX_INPUT_GPX_DIR);
    final File expectedResultsDir = new File(GENERAL_GPX_EXPECTED_RESULTS_DIR);
    final List<URL> expectedResultsResources = new ArrayList<>();
    final Map<String, URL> baseNameToExpectedResultURL = new HashMap<>();
    for (final File file : expectedResultsDir.listFiles(new FileFilter() {

        @Override
        public boolean accept(final File pathname) {
            final Map<String, Object> map = new HashMap<>();
            try {
                map.put("url", pathname.toURI().toURL());
                return DataStoreFinder.getDataStore(map) != null;
            } catch (final IOException e) {
                LOGGER.warn("Cannot read file as GeoTools data store", e);
            }
            return false;
        }
    })) {
        baseNameToExpectedResultURL.put(FilenameUtils.getBaseName(file.getName()).replaceAll("_filtered", ""), file.toURI().toURL());
    }
    for (final String filename : gpxInputDir.list(new FilenameFilter() {

        @Override
        public boolean accept(final File dir, final String name) {
            return FilenameUtils.isExtension(name, new GpxIngestPlugin().getFileExtensionFilters());
        }
    })) {
        final URL url = baseNameToExpectedResultURL.get(FilenameUtils.getBaseName(filename));
        Assert.assertNotNull(url);
        expectedResultsResources.add(url);
    }
    final ExpectedResults expectedResults = TestUtils.getExpectedResults(expectedResultsResources.toArray(new URL[expectedResultsResources.size()]));
    runTestJob(expectedResults, TestUtils.resourceToQuery(new File(GENERAL_GPX_FILTER_FILE).toURI().toURL()), null, null);
}

Example 3 with ExpectedResults

use of org.locationtech.geowave.test.TestUtils.ExpectedResults in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testQuery.

protected void testQuery(final URL savedFilterResource, final URL[] expectedResultsResources, final Pair<String, String> optimalCqlQueryGeometryAndTimeFields, final Index index, final String queryDescription, final CoordinateReferenceSystem crs, final boolean countDuplicates) throws IOException {
    LOGGER.info("querying " + queryDescription);
    final DataStore geowaveStore = getDataStorePluginOptions().createDataStore();
    // this file is the filtered dataset (using the previous file as a
    // filter) so use it to ensure the query worked
    final QueryConstraints constraints = TestUtils.resourceToQuery(savedFilterResource, optimalCqlQueryGeometryAndTimeFields, true);
    QueryBuilder<?, ?> bldr = QueryBuilder.newBuilder();
    if (index != null) {
        bldr = bldr.indexName(index.getName());
    }
    try (final CloseableIterator<?> actualResults = geowaveStore.query(bldr.constraints(constraints).build())) {
        final ExpectedResults expectedResults = TestUtils.getExpectedResults(expectedResultsResources, crs);
        int totalResults = 0;
        final List<Long> actualCentroids = new ArrayList<>();
        while (actualResults.hasNext()) {
            final Object obj = actualResults.next();
            if (obj instanceof SimpleFeature) {
                final SimpleFeature result = (SimpleFeature) obj;
                final long actualHashCentroid = TestUtils.hashCentroid((Geometry) result.getDefaultGeometry());
                Assert.assertTrue("Actual result '" + result.toString() + "' not found in expected result set", expectedResults.hashedCentroids.contains(actualHashCentroid));
                actualCentroids.add(actualHashCentroid);
                totalResults++;
            } else {
                TestUtils.deleteAll(getDataStorePluginOptions());
                Assert.fail("Actual result '" + obj.toString() + "' is not of type Simple Feature.");
            }
        }
        for (final long l : actualCentroids) {
            expectedResults.hashedCentroids.remove(l);
        }
        for (final long l : expectedResults.hashedCentroids) {
            LOGGER.error("Missing expected hashed centroid: " + l);
        }
        if (expectedResults.count != totalResults) {
            TestUtils.deleteAll(getDataStorePluginOptions());
        }
        Assert.assertEquals(expectedResults.count, totalResults);
        final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
        long statisticsResult = 0;
        int duplicates = 0;
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
            AggregationQueryBuilder<?, Long, ?, ?> aggBldr = AggregationQueryBuilder.newBuilder();
            if (index != null) {
                aggBldr = aggBldr.indexName(index.getName());
            }
            aggBldr = aggBldr.constraints(constraints);
            if (countDuplicates) {
                aggBldr.aggregate(internalDataAdapter.getTypeName(), (Aggregation) new DuplicateCountAggregation());
                final DuplicateCount countResult = (DuplicateCount) geowaveStore.aggregate((AggregationQuery) aggBldr.build());
                if (countResult != null) {
                    duplicates += countResult.count;
                }
            }
            aggBldr.count(internalDataAdapter.getTypeName());
            final Long countResult = geowaveStore.aggregate(aggBldr.build());
            // results should already be aggregated, there should be
            // exactly one value in this iterator
            Assert.assertNotNull(countResult);
            statisticsResult += countResult;
        }
        Assert.assertEquals(expectedResults.count, statisticsResult - duplicates);
    }
}

Example 4 with ExpectedResults

use of org.locationtech.geowave.test.TestUtils.ExpectedResults in project geowave by locationtech.

the class SparkUtils method verifyQuery.

public static void verifyQuery(final DataStorePluginOptions dataStore, final SparkContext context, final URL filterFile, final URL[] expectedResultsFiles, final String name, final CoordinateReferenceSystem crsTransform, final Pair<String, String> optimalCqlQueryGeometryAndTimeFields, final boolean useDuring) {
    try {
        // get expected results
        final ExpectedResults expectedResults = TestUtils.getExpectedResults(expectedResultsFiles);
        QueryConstraints query;
        if (crsTransform != null) {
            final SimpleFeature feature = TestUtils.resourceToFeature(filterFile);
            query = TestUtils.featureToQuery(GeometryUtils.crsTransform(feature, SimpleFeatureTypeBuilder.retype(feature.getFeatureType(), crsTransform), CRS.findMathTransform(GeometryUtils.getDefaultCRS(), crsTransform, true)), null, GeometryUtils.getCrsCode(crsTransform), useDuring);
        } else {
            query = TestUtils.resourceToQuery(filterFile, optimalCqlQueryGeometryAndTimeFields, useDuring);
        }
        // Load RDD using spatial query (bbox)
        final RDDOptions queryOpts = new RDDOptions();
        queryOpts.setQuery(QueryBuilder.newBuilder().constraints(query).build());
        queryOpts.setMinSplits(DEFAULT_SPLITS_FOR_COUNT);
        queryOpts.setMaxSplits(DEFAULT_SPLITS_FOR_COUNT);
        final GeoWaveRDD newRDD = GeoWaveRDDLoader.loadRDD(context, dataStore, queryOpts);
        final JavaPairRDD<GeoWaveInputKey, SimpleFeature> javaRdd = newRDD.getRawRDD();
        final long count = getCount(javaRdd, dataStore.getType());
        LOGGER.warn("DataStore loaded into RDD with " + count + " features.");
        // Verify RDD count matches expected count
        Assert.assertEquals(expectedResults.count, count);
    } catch (final Exception e) {
        e.printStackTrace();
        TestUtils.deleteAll(dataStore);
        Assert.fail("Error occurred while testing '" + name + "'");
    }
}