Examples with GpxIngestPlugin org.locationtech.geowave.format.gpx.GpxIngestPlugin used on opensource projects

Example 1 with GpxIngestPlugin

use of org.locationtech.geowave.format.gpx.GpxIngestPlugin in project geowave by locationtech.

the class BasicMapReduceIT method testIngestOsmGpxMultipleIndices.

@Test
public void testIngestOsmGpxMultipleIndices() throws Exception {
    TestUtils.deleteAll(dataStorePluginOptions);
    // ingest the data set into multiple indices and then try several query
    // methods, by adapter and by index
    MapReduceTestUtils.testMapReduceIngest(dataStorePluginOptions, DimensionalityType.SPATIAL_AND_SPATIAL_TEMPORAL, OSM_GPX_INPUT_DIR);
    final DataTypeAdapter<SimpleFeature>[] adapters = new GpxIngestPlugin().getDataAdapters(null);
    final org.locationtech.geowave.core.store.api.DataStore geowaveStore = dataStorePluginOptions.createDataStore();
    final Map<String, ExpectedResults> adapterIdToResultsMap = new HashMap<>();
    for (final DataTypeAdapter<SimpleFeature> adapter : adapters) {
        adapterIdToResultsMap.put(adapter.getTypeName(), TestUtils.getExpectedResults(geowaveStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).build())));
    }
    final List<DataTypeAdapter<?>> firstTwoAdapters = new ArrayList<>();
    firstTwoAdapters.add(adapters[0]);
    firstTwoAdapters.add(adapters[1]);
    final ExpectedResults firstTwoAdaptersResults = TestUtils.getExpectedResults(geowaveStore.query(QueryBuilder.newBuilder().addTypeName(adapters[0].getTypeName()).addTypeName(adapters[1].getTypeName()).build()));
    final ExpectedResults fullDataSetResults = TestUtils.getExpectedResults(geowaveStore.query(QueryBuilder.newBuilder().build()));
    // just for sanity verify its greater than 0 (ie. that data was actually
    // ingested in the first place)
    Assert.assertTrue("There is no data ingested from OSM GPX test files", fullDataSetResults.count > 0);
    // now that we have expected results, run map-reduce export and
    // re-ingest it
    testMapReduceExportAndReingest(DimensionalityType.SPATIAL_AND_SPATIAL_TEMPORAL);
    // first try each adapter individually
    for (final DataTypeAdapter<SimpleFeature> adapter : adapters) {
        final ExpectedResults expResults = adapterIdToResultsMap.get(adapter.getTypeName());
        if (expResults.count > 0) {
            LOGGER.error("Running test for adapter " + adapter.getTypeName());
            runTestJob(expResults, null, new DataTypeAdapter[] { adapter }, null);
        }
    }
    // then try the first 2 adapters, and may as well try with both indices
    // set (should be the default behavior anyways)
    runTestJob(firstTwoAdaptersResults, null, new DataTypeAdapter[] { adapters[0], adapters[1] }, null);
    // now try all adapters and the spatial temporal index, the result
    // should be the full data set
    runTestJob(fullDataSetResults, null, adapters, TestUtils.DEFAULT_SPATIAL_TEMPORAL_INDEX);
    // and finally run with nothing set, should be the full data set
    runTestJob(fullDataSetResults, null, null, null);
}

Example 2 with GpxIngestPlugin

use of org.locationtech.geowave.format.gpx.GpxIngestPlugin in project geowave by locationtech.

the class BasicMapReduceIT method testIngestAndQueryGeneralGpx.

@Test
public void testIngestAndQueryGeneralGpx() throws Exception {
    TestUtils.deleteAll(dataStorePluginOptions);
    MapReduceTestUtils.testMapReduceIngest(dataStorePluginOptions, DimensionalityType.SPATIAL, GENERAL_GPX_INPUT_GPX_DIR);
    final File gpxInputDir = new File(GENERAL_GPX_INPUT_GPX_DIR);
    final File expectedResultsDir = new File(GENERAL_GPX_EXPECTED_RESULTS_DIR);
    final List<URL> expectedResultsResources = new ArrayList<>();
    final Map<String, URL> baseNameToExpectedResultURL = new HashMap<>();
    for (final File file : expectedResultsDir.listFiles(new FileFilter() {

        @Override
        public boolean accept(final File pathname) {
            final Map<String, Object> map = new HashMap<>();
            try {
                map.put("url", pathname.toURI().toURL());
                return DataStoreFinder.getDataStore(map) != null;
            } catch (final IOException e) {
                LOGGER.warn("Cannot read file as GeoTools data store", e);
            }
            return false;
        }
    })) {
        baseNameToExpectedResultURL.put(FilenameUtils.getBaseName(file.getName()).replaceAll("_filtered", ""), file.toURI().toURL());
    }
    for (final String filename : gpxInputDir.list(new FilenameFilter() {

        @Override
        public boolean accept(final File dir, final String name) {
            return FilenameUtils.isExtension(name, new GpxIngestPlugin().getFileExtensionFilters());
        }
    })) {
        final URL url = baseNameToExpectedResultURL.get(FilenameUtils.getBaseName(filename));
        Assert.assertNotNull(url);
        expectedResultsResources.add(url);
    }
    final ExpectedResults expectedResults = TestUtils.getExpectedResults(expectedResultsResources.toArray(new URL[expectedResultsResources.size()]));
    runTestJob(expectedResults, TestUtils.resourceToQuery(new File(GENERAL_GPX_FILTER_FILE).toURI().toURL()), null, null);
}

Example 3 with GpxIngestPlugin

use of org.locationtech.geowave.format.gpx.GpxIngestPlugin in project geowave by locationtech.

the class GPXIngestPluginTest method test.

@Test
public void test() throws IOException {
    final Set<String> expectedSet = HelperClass.buildSet(expectedResults);
    final GpxIngestPlugin pluggin = new GpxIngestPlugin();
    pluggin.init(new File(this.getClass().getClassLoader().getResource("metadata.xml").getPath()).getParentFile().toURI().toURL());
    final CloseableIterator<GeoWaveData<SimpleFeature>> consumer = pluggin.toGeoWaveData(this.getClass().getClassLoader().getResource("12345.xml"), new String[] { "123" });
    int totalCount = 0;
    while (consumer.hasNext()) {
        final GeoWaveData<SimpleFeature> data = consumer.next();
        expectedSet.remove(data.getValue().getID());
        final ValidateObject<SimpleFeature> tester = expectedResults.get(data.getValue().getID());
        if (tester != null) {
            assertTrue(data.getValue().toString(), tester.validate(data.getValue()));
        }
        totalCount++;
    }
    consumer.close();
    assertEquals(9, totalCount);
    // did everything get validated?
    if (expectedSet.size() > 0) {
        System.out.println("Failed matches:");
        System.out.println(expectedSet);
    }
    assertEquals("All expected data set should be matched; zero unmatched data expected", 0, expectedSet.size());
}