Examples with GeoEntryExtractor org.codice.ddf.spatial.geocoding.GeoEntryExtractor used on opensource projects

Example 6 with GeoEntryExtractor

use of org.codice.ddf.spatial.geocoding.GeoEntryExtractor in project ddf by codice.

the class GazetteerUpdateCommandTest method testProgressOutput.

@Test
public void testProgressOutput() throws Exception {
    final GeoEntryExtractor geoEntryExtractor = spy(new GeoEntryExtractor() {

        @Override
        public List<GeoEntry> getGeoEntries(final String resource, final ProgressCallback progressCallback) {
            return null;
        }

        @Override
        public void pushGeoEntriesToExtractionCallback(final String resource, final ExtractionCallback extractionCallback) {
            extractionCallback.updateProgress(50);
            assertThat(consoleInterceptor.getOutput(), containsString("50%"));
            extractionCallback.updateProgress(100);
            assertThat(consoleInterceptor.getOutput(), containsString("100%"));
        }

        @Override
        public void setUrl(String url) {
            return;
        }
    });
    final GeoEntryExtractor geoEntryUrlExtractor = spy(new GeoEntryExtractor() {

        @Override
        public List<GeoEntry> getGeoEntries(final String resource, final ProgressCallback progressCallback) {
            return null;
        }

        @Override
        public void pushGeoEntriesToExtractionCallback(final String resource, final ExtractionCallback extractionCallback) {
            extractionCallback.updateProgress(50);
            assertThat(consoleInterceptor.getOutput(), containsString("50%"));
            extractionCallback.updateProgress(100);
            assertThat(consoleInterceptor.getOutput(), containsString("100%"));
        }

        @Override
        public void setUrl(String url) {
            return;
        }
    });
    final GeoEntryIndexer geoEntryIndexer = spy(new GeoEntryIndexer() {

        @Override
        public void updateIndex(final List<GeoEntry> newEntries, final boolean create, final ProgressCallback progressCallback, final String entrySource) {
        /* stub */
        }

        @Override
        public void updateIndex(final String resource, final GeoEntryExtractor geoEntryExtractor, final boolean create, final ProgressCallback progressCallback) throws GeoNamesRemoteDownloadException, GeoEntryIndexingException, GeoEntryExtractionException {
            final ExtractionCallback extractionCallback = new ExtractionCallback() {

                @Override
                public void extracted(final GeoEntry newEntry) {
                // Not used in test
                }

                @Override
                public void updateProgress(final int progress) {
                    progressCallback.updateProgress(progress);
                }
            };
            geoEntryExtractor.pushGeoEntriesToExtractionCallback(resource, extractionCallback);
        }
    });
    List<GeoEntryExtractor> geoEntryExtractors = new ArrayList<>();
    geoEntryExtractors.add(geoEntryExtractor);
    geoEntryExtractors.add(geoEntryUrlExtractor);
    gazetteerUpdateCommand.setGeoEntryExtractor(geoEntryExtractor);
    gazetteerUpdateCommand.setGeoEntryIndexer(geoEntryIndexer);
    gazetteerUpdateCommand.setResource("test");
    gazetteerUpdateCommand.executeWithSubject();
    consoleInterceptor.resetSystemOut();
    consoleInterceptor.closeBuffer();
}

Example 7 with GeoEntryExtractor

use of org.codice.ddf.spatial.geocoding.GeoEntryExtractor in project ddf by codice.

the class TestGeoNamesUpdateCommand method testExceptionDuringExtraction.

@Test
public void testExceptionDuringExtraction() throws IOException, GeoNamesRemoteDownloadException, GeoEntryExtractionException, GeoEntryIndexingException {
    final String errorText = "Extraction error text";
    final GeoEntryExtractor geoEntryExtractor = mock(GeoEntryExtractor.class);
    final GeoEntryExtractionException geoEntryExtractionException = new GeoEntryExtractionException(errorText);
    doThrow(geoEntryExtractionException).when(geoEntryExtractor).pushGeoEntriesToExtractionCallback(anyString(), any(ExtractionCallback.class));
    final GeoEntryIndexer geoEntryIndexer = new GeoEntryIndexer() {

        @Override
        public void updateIndex(final List<GeoEntry> newEntries, final boolean create, final ProgressCallback progressCallback) {
        }

        @Override
        public void updateIndex(final String resource, final GeoEntryExtractor geoEntryExtractor, final boolean create, final ProgressCallback progressCallback) throws GeoNamesRemoteDownloadException, GeoEntryExtractionException, GeoEntryIndexingException {
            geoEntryExtractor.pushGeoEntriesToExtractionCallback(resource, mock(ExtractionCallback.class));
        }
    };
    geoNamesUpdateCommand.setGeoEntryIndexer(geoEntryIndexer);
    geoNamesUpdateCommand.setGeoEntryExtractor(geoEntryExtractor);
    geoNamesUpdateCommand.setResource("temp.txt");
    geoNamesUpdateCommand.execute();
    assertThat(consoleInterceptor.getOutput(), containsString(errorText));
    consoleInterceptor.resetSystemOut();
    consoleInterceptor.closeBuffer();
}

Example 8 with GeoEntryExtractor

use of org.codice.ddf.spatial.geocoding.GeoEntryExtractor in project ddf by codice.

the class TestGeoNamesUpdateCommand method testProgressOutput.

@Test
public void testProgressOutput() throws IOException {
    final GeoEntryExtractor geoEntryExtractor = spy(new GeoEntryExtractor() {

        @Override
        public List<GeoEntry> getGeoEntries(final String resource, final ProgressCallback progressCallback) {
            return null;
        }

        @Override
        public void pushGeoEntriesToExtractionCallback(final String resource, final ExtractionCallback extractionCallback) {
            extractionCallback.updateProgress(50);
            assertThat(consoleInterceptor.getOutput(), containsString("50%"));
            extractionCallback.updateProgress(100);
            assertThat(consoleInterceptor.getOutput(), containsString("100%"));
        }

        @Override
        public void setUrl(String url) {
            return;
        }
    });
    final GeoEntryExtractor geoEntryUrlExtractor = spy(new GeoEntryExtractor() {

        @Override
        public List<GeoEntry> getGeoEntries(final String resource, final ProgressCallback progressCallback) {
            return null;
        }

        @Override
        public void pushGeoEntriesToExtractionCallback(final String resource, final ExtractionCallback extractionCallback) {
            extractionCallback.updateProgress(50);
            assertThat(consoleInterceptor.getOutput(), containsString("50%"));
            extractionCallback.updateProgress(100);
            assertThat(consoleInterceptor.getOutput(), containsString("100%"));
        }

        @Override
        public void setUrl(String url) {
            return;
        }
    });
    final GeoEntryIndexer geoEntryIndexer = spy(new GeoEntryIndexer() {

        @Override
        public void updateIndex(final List<GeoEntry> newEntries, final boolean create, final ProgressCallback progressCallback) {
        }

        @Override
        public void updateIndex(final String resource, final GeoEntryExtractor geoEntryExtractor, final boolean create, final ProgressCallback progressCallback) throws GeoNamesRemoteDownloadException, GeoEntryIndexingException, GeoEntryExtractionException {
            final ExtractionCallback extractionCallback = new ExtractionCallback() {

                @Override
                public void extracted(final GeoEntry newEntry) {
                }

                @Override
                public void updateProgress(final int progress) {
                    progressCallback.updateProgress(progress);
                }
            };
            geoEntryExtractor.pushGeoEntriesToExtractionCallback(resource, extractionCallback);
        }
    });
    List<GeoEntryExtractor> geoEntryExtractors = new ArrayList<GeoEntryExtractor>();
    geoEntryExtractors.add(geoEntryExtractor);
    geoEntryExtractors.add(geoEntryUrlExtractor);
    geoNamesUpdateCommand.setGeoEntryExtractor(geoEntryExtractor);
    geoNamesUpdateCommand.setGeoEntryIndexer(geoEntryIndexer);
    geoNamesUpdateCommand.setResource("test");
    geoNamesUpdateCommand.execute();
    consoleInterceptor.resetSystemOut();
    consoleInterceptor.closeBuffer();
}

Example 9 with GeoEntryExtractor

use of org.codice.ddf.spatial.geocoding.GeoEntryExtractor in project ddf by codice.

the class TestGeoNamesLuceneIndexer method testCreateIndexFromExtractor.

@Test
public void testCreateIndexFromExtractor() throws IOException, GeoEntryIndexingException, GeoNamesRemoteDownloadException, GeoEntryExtractionException {
    configureMocks();
    final ProgressCallback progressCallback = mock(ProgressCallback.class);
    geoNamesLuceneIndexer.updateIndex(null, new GeoEntryExtractor() {

        @Override
        public List<GeoEntry> getGeoEntries(final String resource, final ProgressCallback progressCallback) {
            return null;
        }

        @Override
        public void pushGeoEntriesToExtractionCallback(final String resource, final ExtractionCallback extractionCallback) throws GeoEntryExtractionException {
            try {
                extractionCallback.extracted(GEO_ENTRY_1);
                extractionCallback.extracted(GEO_ENTRY_2);
                extractionCallback.extracted(GEO_ENTRY_3);
                extractionCallback.extracted(GEO_ENTRY_4);
                extractionCallback.extracted(GEO_ENTRY_5);
                extractionCallback.extracted(GEO_ENTRY_6);
                extractionCallback.extracted(GEO_ENTRY_7);
                extractionCallback.extracted(GEO_ENTRY_8);
                extractionCallback.extracted(GEO_ENTRY_9);
                extractionCallback.updateProgress(100);
            } catch (GeoEntryIndexingException e) {
                throw new GeoEntryExtractionException("Unable to add entry.", e);
            }
        }

        @Override
        public void setUrl(String url) {
            return;
        }
    }, true, progressCallback);
    verify(indexWriter, times(9)).addDocument(documentArgumentCaptor.capture());
    final List<Document> documentList = documentArgumentCaptor.getAllValues();
    verifyDocumentList(documentList);
    verify(progressCallback, times(1)).updateProgress(100);
}

Example 10 with GeoEntryExtractor

use of org.codice.ddf.spatial.geocoding.GeoEntryExtractor in project ddf by codice.

the class GazetteerUpdateCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    final PrintStream console = System.out;
    final ProgressCallback progressCallback = progress -> {
        console.printf("\r%d%%", progress);
        console.flush();
    };
    final FeatureIndexer.IndexCallback featureIndexCallback = count -> {
        console.printf("\r%d features indexed", count);
        console.flush();
    };
    console.println("Updating...");
    try {
        if (isResourceGeoJSON()) {
            featureIndexer.updateIndex(resource, featureExtractor, create, featureIndexCallback);
        } else {
            geoEntryIndexer.updateIndex(resource, geoEntryExtractor, create, progressCallback);
        }
        console.println("\nDone.");
    } catch (GeoEntryExtractionException | FeatureExtractionException e) {
        LOGGER.info("Error extracting data from resource {}", resource, e);
        console.printf("Could not extract data from resource %s.%n Message: %s%n Check the logs for more details.%n", resource, e.getMessage());
    } catch (GeoEntryIndexingException | FeatureIndexingException e) {
        LOGGER.info("Error indexing data", e);
        console.printf("Could not index the  data.%n Message: %s%n Check the logs for more details.%n", e.getMessage());
    } catch (GeoNamesRemoteDownloadException e) {
        LOGGER.info("Error downloading resource from remote source {}", resource, e);
        console.printf("Could not download the GeoNames file %s.%n  Message: %s%n Check the logs for more details.%n", resource, e.getMessage());
    }
    return null;
}