Search in sources :

Example 1 with StringList

use of org.openstreetmap.atlas.utilities.collections.StringList in project atlas-checks by osmlab.

the class IntegrityCheckSparkJob method start.

@SuppressWarnings({ "rawtypes" })
@Override
public void start(final CommandMap commandMap) {
    final String atlasDirectory = (String) commandMap.get(ATLAS_FOLDER);
    final String input = Optional.ofNullable(input(commandMap)).orElse(atlasDirectory);
    final String output = output(commandMap);
    @SuppressWarnings("unchecked") final Set<OutputFormats> outputFormats = (Set<OutputFormats>) commandMap.get(OUTPUT_FORMATS);
    final StringList countries = StringList.split((String) commandMap.get(COUNTRIES), CommonConstants.COMMA);
    final MapRouletteConfiguration mapRouletteConfiguration = (MapRouletteConfiguration) commandMap.get(MAP_ROULETTE);
    final Configuration checksConfiguration = ConfigurationResolver.loadConfiguration(commandMap, CONFIGURATION_FILES, CONFIGURATION_JSON);
    final boolean saveIntermediateAtlas = (Boolean) commandMap.get(PBF_SAVE_INTERMEDIATE_ATLAS);
    @SuppressWarnings("unchecked") final Rectangle pbfBoundary = ((Optional<Rectangle>) commandMap.getOption(PBF_BOUNDING_BOX)).orElse(Rectangle.MAXIMUM);
    final boolean compressOutput = Boolean.valueOf((String) commandMap.get(SparkJob.COMPRESS_OUTPUT));
    final Map<String, String> sparkContext = configurationMap();
    final CheckResourceLoader checkLoader = new CheckResourceLoader(checksConfiguration);
    // check configuration and country list
    final Set<BaseCheck> preOverriddenChecks = checkLoader.loadChecks();
    if (!isValidInput(countries, preOverriddenChecks)) {
        logger.error("No countries supplied or checks enabled, exiting!");
        return;
    }
    // Read priority countries from the configuration
    final List<String> priorityCountries = checksConfiguration.get("priority.countries", Collections.EMPTY_LIST).value();
    // Create a list of Country to Check tuples
    // Add priority countries first if they are supplied by parameter
    final List<Tuple2<String, Set<BaseCheck>>> countryCheckTuples = new ArrayList<>();
    countries.stream().filter(priorityCountries::contains).forEach(country -> countryCheckTuples.add(new Tuple2<>(country, checkLoader.loadChecksForCountry(country))));
    // Then add the rest of the countries
    countries.stream().filter(country -> !priorityCountries.contains(country)).forEach(country -> countryCheckTuples.add(new Tuple2<>(country, checkLoader.loadChecksForCountry(country))));
    // Log countries and integrity
    logger.info("Initialized countries: {}", countryCheckTuples.stream().map(tuple -> tuple._1).collect(Collectors.joining(",")));
    logger.info("Initialized checks: {}", preOverriddenChecks.stream().map(BaseCheck::getCheckName).collect(Collectors.joining(",")));
    // Parallelize on the countries
    final JavaPairRDD<String, Set<BaseCheck>> countryCheckRDD = getContext().parallelizePairs(countryCheckTuples, countryCheckTuples.size());
    // Set target and temporary folders
    final String targetOutputFolder = SparkFileHelper.parentPath(output);
    final String temporaryOutputFolder = SparkFileHelper.combine(targetOutputFolder, SparkFileHelper.TEMPORARY_FOLDER_NAME);
    // Useful file helper to create/delete/name files and directories
    final SparkFileHelper fileHelper = new SparkFileHelper(sparkContext);
    // Atlas Helper to load different types of Atlas data
    final AtlasDataSource atlasLoader = new AtlasDataSource(sparkContext, checksConfiguration, pbfBoundary);
    // Create target folders
    fileHelper.mkdir(SparkFileHelper.combine(targetOutputFolder, OUTPUT_FLAG_FOLDER));
    fileHelper.mkdir(SparkFileHelper.combine(targetOutputFolder, OUTPUT_GEOJSON_FOLDER));
    fileHelper.mkdir(SparkFileHelper.combine(targetOutputFolder, OUTPUT_METRIC_FOLDER));
    // Run the set of flags per country per check. The output will be an RDD pair mapping each
    // country with a set of SparkFilePaths to flags, geojson and metrics generated.
    final JavaPairRDD<String, Set<SparkFilePath>> resultRDD = countryCheckRDD.mapToPair(tuple -> {
        final Time timer = Time.now();
        final String country = tuple._1();
        final Set<BaseCheck> checks = tuple._2();
        logger.info("Initialized checks for {}: {}", country, checks.stream().map(BaseCheck::getCheckName).collect(Collectors.joining(",")));
        final Set<SparkFilePath> resultingFiles = new HashSet<>();
        final SparkFilePath flagOutput;
        if (outputFormats.contains(OutputFormats.FLAGS)) {
            // Initialize flag output processor
            flagOutput = initializeOutput(OUTPUT_FLAG_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            EventService.get(country).register(new CheckFlagFileProcessor(fileHelper, flagOutput.getTemporaryPath()).withCompression(compressOutput));
        } else {
            flagOutput = null;
        }
        final SparkFilePath geoJsonOutput;
        if (outputFormats.contains(OutputFormats.GEOJSON)) {
            // Initialize geojson output processor
            geoJsonOutput = initializeOutput(OUTPUT_GEOJSON_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            EventService.get(country).register(new CheckFlagGeoJsonProcessor(fileHelper, geoJsonOutput.getTemporaryPath()).withCompression(compressOutput));
        } else {
            geoJsonOutput = null;
        }
        final SparkFilePath metricOutput;
        if (outputFormats.contains(OutputFormats.METRICS)) {
            // Initialize metric output processor
            metricOutput = initializeOutput(OUTPUT_METRIC_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            EventService.get(country).register(new MetricFileGenerator(METRICS_FILENAME, fileHelper, metricOutput.getTemporaryPath()));
        } else {
            metricOutput = null;
        }
        final Consumer<Atlas> intermediateAtlasHandler;
        if (saveIntermediateAtlas) {
            final SparkFilePath atlasOutput = initializeOutput(OUTPUT_ATLAS_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            intermediateAtlasHandler = atlas -> {
                writeAtlas(atlas, country, atlasOutput, fileHelper);
                resultingFiles.add(atlasOutput);
            };
        } else {
            intermediateAtlasHandler = atlas -> {
            // no-op
            };
        }
        try {
            final Atlas atlas = atlasLoader.load(input, country, intermediateAtlasHandler);
            if (atlas == null) {
                logger.error("Could not find {} Atlas files. Skipping country!", country);
            } else {
                executeChecks(country, atlas, checks, mapRouletteConfiguration);
                // Add output folders for handling later
                Stream.of(flagOutput, metricOutput, geoJsonOutput).filter(Objects::nonNull).forEach(resultingFiles::add);
            }
            EventService.get(country).complete();
            return new Tuple2<>(country, resultingFiles);
        } catch (final CoreException e) {
            logger.error("Exception running integrity checks on {}", country, e);
        }
        logger.trace("Integrity checks took {} ms to execute for {}.", timer.elapsedSince().asMilliseconds(), country);
        return new Tuple2<>(IGNORED_KEY, null);
    }).filter(tuple -> !tuple._1().equals(IGNORED_KEY));
    // Commit results
    resultRDD.foreach(countryPathPair -> {
        final String country = countryPathPair._1();
        final Set<SparkFilePath> paths = countryPathPair._2();
        logger.info("[{}] Committing outputs: {}", country, paths);
        paths.forEach(fileHelper::commit);
    });
    try {
        // Clean up
        logger.info("Deleting {}.", temporaryOutputFolder);
        fileHelper.deleteDirectory(temporaryOutputFolder);
    } catch (final Exception e) {
        logger.warn("Clean up failed!", e);
    }
}
Also used : SparkJob(org.openstreetmap.atlas.generator.tools.spark.SparkJob) Atlas(org.openstreetmap.atlas.geography.atlas.Atlas) CommonConstants(org.openstreetmap.atlas.checks.constants.CommonConstants) CheckResourceLoader(org.openstreetmap.atlas.checks.base.CheckResourceLoader) SparkFilePath(org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFilePath) LoggerFactory(org.slf4j.LoggerFactory) MapRouletteConfiguration(org.openstreetmap.atlas.checks.maproulette.MapRouletteConfiguration) Relation(org.openstreetmap.atlas.geography.atlas.items.Relation) Finder(org.openstreetmap.atlas.geography.atlas.items.complex.Finder) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) BaseCheck(org.openstreetmap.atlas.checks.base.BaseCheck) MapRouletteClient(org.openstreetmap.atlas.checks.maproulette.MapRouletteClient) AtlasEntity(org.openstreetmap.atlas.geography.atlas.items.AtlasEntity) Configuration(org.openstreetmap.atlas.utilities.configuration.Configuration) ComplexEntity(org.openstreetmap.atlas.geography.atlas.items.complex.ComplexEntity) StringConverter(org.openstreetmap.atlas.utilities.conversion.StringConverter) Map(java.util.Map) ConfigurationResolver(org.openstreetmap.atlas.checks.configuration.ConfigurationResolver) CheckFlagGeoJsonProcessor(org.openstreetmap.atlas.checks.event.CheckFlagGeoJsonProcessor) Duration(org.openstreetmap.atlas.utilities.scalars.Duration) Logger(org.slf4j.Logger) TaskContext(org.apache.spark.TaskContext) CoreException(org.openstreetmap.atlas.exception.CoreException) SparkFileOutput(org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileOutput) Set(java.util.Set) CommandMap(org.openstreetmap.atlas.utilities.runtime.CommandMap) CheckFlagFileProcessor(org.openstreetmap.atlas.checks.event.CheckFlagFileProcessor) MetricFileGenerator(org.openstreetmap.atlas.checks.event.MetricFileGenerator) Tuple2(scala.Tuple2) Collectors(java.util.stream.Collectors) JavaPairRDD(org.apache.spark.api.java.JavaPairRDD) Pool(org.openstreetmap.atlas.utilities.threads.Pool) Objects(java.util.Objects) Consumer(java.util.function.Consumer) FileSuffix(org.openstreetmap.atlas.streaming.resource.FileSuffix) List(java.util.List) Stream(java.util.stream.Stream) Iterables(org.openstreetmap.atlas.utilities.collections.Iterables) EventService(org.openstreetmap.atlas.checks.event.EventService) SparkFileHelper(org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileHelper) MultiIterable(org.openstreetmap.atlas.utilities.collections.MultiIterable) Optional(java.util.Optional) Rectangle(org.openstreetmap.atlas.geography.Rectangle) StringList(org.openstreetmap.atlas.utilities.collections.StringList) Time(org.openstreetmap.atlas.utilities.time.Time) Collections(java.util.Collections) HashSet(java.util.HashSet) Set(java.util.Set) MapRouletteConfiguration(org.openstreetmap.atlas.checks.maproulette.MapRouletteConfiguration) Configuration(org.openstreetmap.atlas.utilities.configuration.Configuration) Rectangle(org.openstreetmap.atlas.geography.Rectangle) ArrayList(java.util.ArrayList) CheckFlagGeoJsonProcessor(org.openstreetmap.atlas.checks.event.CheckFlagGeoJsonProcessor) BaseCheck(org.openstreetmap.atlas.checks.base.BaseCheck) Time(org.openstreetmap.atlas.utilities.time.Time) Consumer(java.util.function.Consumer) CheckResourceLoader(org.openstreetmap.atlas.checks.base.CheckResourceLoader) Atlas(org.openstreetmap.atlas.geography.atlas.Atlas) Optional(java.util.Optional) StringList(org.openstreetmap.atlas.utilities.collections.StringList) CoreException(org.openstreetmap.atlas.exception.CoreException) CheckFlagFileProcessor(org.openstreetmap.atlas.checks.event.CheckFlagFileProcessor) MapRouletteConfiguration(org.openstreetmap.atlas.checks.maproulette.MapRouletteConfiguration) CoreException(org.openstreetmap.atlas.exception.CoreException) SparkFilePath(org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFilePath) Tuple2(scala.Tuple2) MetricFileGenerator(org.openstreetmap.atlas.checks.event.MetricFileGenerator) Objects(java.util.Objects) SparkFileHelper(org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileHelper)

Example 2 with StringList

use of org.openstreetmap.atlas.utilities.collections.StringList in project atlas-checks by osmlab.

the class ConfigurationResolver method loadConfiguration.

/**
 * Resolves a {@link Configuration} for {@link Command}s given the {@link CommandMap} and
 * {@link Command.Switch}s available
 *
 * @param commandMap
 *            the {@link Command}'s {@link CommandMap}
 * @param keyFiles
 *            the {@link Command.Switch} containing a list of URIs for configuration
 * @param keyJson
 *            the {@link Command.Switch} containing inline JSON configuration
 * @return a new {@link Configuration}
 */
public static Configuration loadConfiguration(final CommandMap commandMap, final Command.Switch<StringList> keyFiles, final Command.Switch<String> keyJson) {
    final List<InputStream> configurationSources = new ArrayList<>();
    ConfigurationResolver.getResourceAsStream("application.json").ifPresent(configurationSources::add);
    commandMap.getOption(keyFiles).ifPresent(files -> ((StringList) files).forEach(uri -> ConfigurationResolver.getResourceAsStream(URI.create(uri)).ifPresent(configurationSources::add)));
    commandMap.getOption(keyJson).map(value -> new ByteArrayInputStream(value.toString().getBytes(StandardCharsets.UTF_8))).ifPresent(configurationSources::add);
    final List<Resource> configurationResources = configurationSources.stream().map(InputStreamResource::new).collect(Collectors.toList());
    final Configuration configuration;
    Throwable thrown = null;
    try {
        configuration = new MergedConfiguration(Iterables.head(configurationResources), Iterables.tail(configurationResources));
    } finally {
        for (final InputStream source : configurationSources) {
            try {
                source.close();
            } catch (final Throwable throwable) {
                thrown = throwable;
            }
        }
    }
    if (thrown != null) {
        throw new CoreException("Failed to load configuration", thrown);
    }
    return configuration;
}
Also used : BufferedInputStream(java.io.BufferedInputStream) StandardConfiguration(org.openstreetmap.atlas.utilities.configuration.StandardConfiguration) LoggerFactory(org.slf4j.LoggerFactory) HttpStatus(org.apache.http.HttpStatus) ArrayList(java.util.ArrayList) Configuration(org.openstreetmap.atlas.utilities.configuration.Configuration) ByteArrayInputStream(java.io.ByteArrayInputStream) URI(java.net.URI) Command(org.openstreetmap.atlas.utilities.runtime.Command) File(org.openstreetmap.atlas.streaming.resource.File) MergedConfiguration(org.openstreetmap.atlas.utilities.configuration.MergedConfiguration) Logger(org.slf4j.Logger) Resource(org.openstreetmap.atlas.streaming.resource.Resource) CoreException(org.openstreetmap.atlas.exception.CoreException) CommandMap(org.openstreetmap.atlas.utilities.runtime.CommandMap) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) GetResource(org.openstreetmap.atlas.streaming.resource.http.GetResource) List(java.util.List) Iterables(org.openstreetmap.atlas.utilities.collections.Iterables) InputStreamResource(org.openstreetmap.atlas.streaming.resource.InputStreamResource) Optional(java.util.Optional) StringResource(org.openstreetmap.atlas.streaming.resource.StringResource) StringList(org.openstreetmap.atlas.utilities.collections.StringList) Assert(org.junit.Assert) InputStream(java.io.InputStream) MergedConfiguration(org.openstreetmap.atlas.utilities.configuration.MergedConfiguration) StandardConfiguration(org.openstreetmap.atlas.utilities.configuration.StandardConfiguration) Configuration(org.openstreetmap.atlas.utilities.configuration.Configuration) MergedConfiguration(org.openstreetmap.atlas.utilities.configuration.MergedConfiguration) CoreException(org.openstreetmap.atlas.exception.CoreException) ByteArrayInputStream(java.io.ByteArrayInputStream) BufferedInputStream(java.io.BufferedInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) ArrayList(java.util.ArrayList) Resource(org.openstreetmap.atlas.streaming.resource.Resource) GetResource(org.openstreetmap.atlas.streaming.resource.http.GetResource) InputStreamResource(org.openstreetmap.atlas.streaming.resource.InputStreamResource) StringResource(org.openstreetmap.atlas.streaming.resource.StringResource)

Aggregations

ArrayList (java.util.ArrayList)2 List (java.util.List)2 Optional (java.util.Optional)2 Collectors (java.util.stream.Collectors)2 CoreException (org.openstreetmap.atlas.exception.CoreException)2 Iterables (org.openstreetmap.atlas.utilities.collections.Iterables)2 StringList (org.openstreetmap.atlas.utilities.collections.StringList)2 Configuration (org.openstreetmap.atlas.utilities.configuration.Configuration)2 CommandMap (org.openstreetmap.atlas.utilities.runtime.CommandMap)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2 BufferedInputStream (java.io.BufferedInputStream)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 URI (java.net.URI)1 StandardCharsets (java.nio.charset.StandardCharsets)1 Collections (java.util.Collections)1 HashSet (java.util.HashSet)1 Map (java.util.Map)1