Examples with Resource org.openstreetmap.atlas.streaming.resource.Resource used on opensource projects

Example 1 with Resource

use of org.openstreetmap.atlas.streaming.resource.Resource in project atlas-checks by osmlab.

the class ConfigurationResolver method loadConfiguration.

/**
 * Resolves a {@link Configuration} for {@link Command}s given the {@link CommandMap} and
 * {@link Command.Switch}s available
 *
 * @param commandMap
 *            the {@link Command}'s {@link CommandMap}
 * @param keyFiles
 *            the {@link Command.Switch} containing a list of URIs for configuration
 * @param keyJson
 *            the {@link Command.Switch} containing inline JSON configuration
 * @return a new {@link Configuration}
 */
public static Configuration loadConfiguration(final CommandMap commandMap, final Command.Switch<StringList> keyFiles, final Command.Switch<String> keyJson) {
    final List<InputStream> configurationSources = new ArrayList<>();
    ConfigurationResolver.getResourceAsStream("application.json").ifPresent(configurationSources::add);
    commandMap.getOption(keyFiles).ifPresent(files -> ((StringList) files).forEach(uri -> ConfigurationResolver.getResourceAsStream(URI.create(uri)).ifPresent(configurationSources::add)));
    commandMap.getOption(keyJson).map(value -> new ByteArrayInputStream(value.toString().getBytes(StandardCharsets.UTF_8))).ifPresent(configurationSources::add);
    final List<Resource> configurationResources = configurationSources.stream().map(InputStreamResource::new).collect(Collectors.toList());
    final Configuration configuration;
    Throwable thrown = null;
    try {
        configuration = new MergedConfiguration(Iterables.head(configurationResources), Iterables.tail(configurationResources));
    } finally {
        for (final InputStream source : configurationSources) {
            try {
                source.close();
            } catch (final Throwable throwable) {
                thrown = throwable;
            }
        }
    }
    if (thrown != null) {
        throw new CoreException("Failed to load configuration", thrown);
    }
    return configuration;
}

Example 2 with Resource

use of org.openstreetmap.atlas.streaming.resource.Resource in project atlas-checks by osmlab.

the class CheckFlagFileProcessorTest method processCompleteAndValidate.

private void processCompleteAndValidate(final int eventCount) {
    // Generate
    final File tempDirectory = Files.createTempDir();
    final CheckFlagFileProcessor processor = new CheckFlagFileProcessor(new SparkFileHelper(FILE_SYSTEM_CONFIG), tempDirectory.getAbsolutePath());
    for (int index = 0; index < eventCount; index++) {
        processor.process(SAMPLE_EVENT);
    }
    processor.process(new ShutdownEvent());
    // Validate
    final List<Resource> files = FileSystemHelper.resources(tempDirectory.getAbsolutePath(), FILE_SYSTEM_CONFIG);
    Assert.assertEquals((int) Math.floor(eventCount / (double) BATCH_SIZE) + 1, files.size());
    int actualEventCount = 0;
    for (final Resource file : files) {
        for (final String line : file.lines()) {
            actualEventCount++;
            compareJsonAndEventObject(line, SAMPLE_EVENT);
        }
    }
    Assert.assertEquals(eventCount, actualEventCount);
    // Cleanup
    tempDirectory.delete();
}

Example 3 with Resource

use of org.openstreetmap.atlas.streaming.resource.Resource in project atlas-checks by osmlab.

the class CheckFlagGeoJsonProcessorTest method processCompleteAndValidate.

private void processCompleteAndValidate(final int eventCount) {
    // Generate
    final File tempDirectory = Files.createTempDir();
    final CheckFlagGeoJsonProcessor processor = new CheckFlagGeoJsonProcessor(new SparkFileHelper(FILE_SYSTEM_CONFIG), tempDirectory.getAbsolutePath()).withBatchSizeOverride(25);
    for (int index = 0; index < eventCount; index++) {
        processor.process(this.setup.getCheckFlagEvent());
    }
    processor.process(new ShutdownEvent());
    // Validate
    final List<Resource> files = FileSystemHelper.resources(tempDirectory.getAbsolutePath(), FILE_SYSTEM_CONFIG);
    Assert.assertEquals((int) Math.ceil(eventCount / (double) processor.computeBatchSize()), files.size());
    int actualEventCount = 0;
    for (final Resource file : files) {
        final JsonObject found = GSON_BUILDER.fromJson(new InputStreamReader(file.read()), JsonObject.class);
        actualEventCount += found.getAsJsonArray("features").size();
    }
    Assert.assertEquals(eventCount, actualEventCount);
    // Cleanup
    tempDirectory.delete();
}

Example 4 with Resource

use of org.openstreetmap.atlas.streaming.resource.Resource in project atlas-checks by osmlab.

the class MetricFileGeneratorTest method processCompleteAndValidate.

private void processCompleteAndValidate(final int eventCount) throws IOException {
    // Generate
    final File tempDirectory = this.tempFolder.newFolder();
    final MetricFileGenerator processor = new MetricFileGenerator("some-file-name.csv", new SparkFileHelper(FILE_SYSTEM_CONFIG), tempDirectory.getAbsolutePath());
    for (int index = 0; index < eventCount; index++) {
        processor.process(SAMPLE_EVENT);
    }
    processor.process(new ShutdownEvent());
    // Validate
    final List<Resource> files = FileSystemHelper.resources(tempDirectory.getAbsolutePath(), FILE_SYSTEM_CONFIG);
    Assert.assertEquals((int) Math.floor(eventCount / (double) BATCH_SIZE) + 1, files.size());
    int actualEventCount = 0;
    for (final Resource file : files) {
        for (final String line : file.lines()) {
            // This is first line
            if (actualEventCount == 0) {
                Assert.assertEquals(MetricEvent.header(), line);
            } else {
                Assert.assertEquals("a-metric-name,60000", line);
            }
            actualEventCount++;
        }
    }
    // Plus 1 is for the header
    Assert.assertEquals(eventCount + 1, actualEventCount);
}

Example 5 with Resource

use of org.openstreetmap.atlas.streaming.resource.Resource in project atlas-checks by osmlab.

the class AtlasDataSource method load.

/**
 * Loads an {@link Atlas} from the input location. Intermediate {@link Atlas}es created are
 * submitted to the provided {@link Consumer} to allow for any additional handling.
 *
 * @param input
 *            location of the {@link Atlas} source
 * @param country
 *            country of the {@link Atlas}
 * @param intermediateAtlasHandler
 *            handler given intermediate {@link Atlas} files when created
 * @return {@link Atlas} representation of the data source
 */
public Atlas load(final String input, final String country, final Consumer<Atlas> intermediateAtlasHandler) {
    // Path filters for supported file types
    final PathFilter pbfFilter = new OsmPbfFilePathFilter();
    final PathFilter atlasFilter = new CountrySpecificAtlasFilePathFilter(country);
    final Optional<Resource> resource = this.loadHelper.collectSourceFile(input, pbfFilter, atlasFilter);
    if (resource.isPresent()) {
        final Resource dataSource = resource.get();
        if (Atlas.isAtlas(dataSource)) {
            return new AtlasResourceLoader().load(dataSource);
        } else if (FileSuffix.resourceFilter(FileSuffix.PBF).test(dataSource)) {
            this.logger.info("Loading Atlas from OSM protobuf {}", input);
            final Atlas atlas = this.loadPbf(dataSource, country);
            intermediateAtlasHandler.accept(atlas);
            return atlas;
        }
    } else {
        final String directory = this.pathResolver.resolvePath(input, country);
        final List<Resource> atlasResources = this.loadHelper.collectSourceFiles(directory, true, atlasFilter);
        if (atlasResources.size() > 0) {
            return new AtlasResourceLoader().load(atlasResources);
        } else {
            final List<Resource> pbfResources = this.loadHelper.collectSourceFiles(directory, true, pbfFilter);
            final int pbfCount = pbfResources.size();
            if (pbfCount > 0) {
                this.logger.info("Loading Atlas from {} OSM protobuf(s) found in {}", pbfCount, input);
                final List<Atlas> atlases = pbfResources.parallelStream().map(dataSource -> this.loadPbf(dataSource, country)).peek(intermediateAtlasHandler).collect(Collectors.toList());
                return new MultiAtlas(atlases);
            }
        }
    }
    return null;
}