Examples with Atlas org.openstreetmap.atlas.geography.atlas.Atlas used on opensource projects

Example 1 with Atlas

use of org.openstreetmap.atlas.geography.atlas.Atlas in project atlas-checks by osmlab.

the class IntegrityCheckSparkJob method start.

@SuppressWarnings({ "rawtypes" })
@Override
public void start(final CommandMap commandMap) {
    final String atlasDirectory = (String) commandMap.get(ATLAS_FOLDER);
    final String input = Optional.ofNullable(input(commandMap)).orElse(atlasDirectory);
    final String output = output(commandMap);
    @SuppressWarnings("unchecked") final Set<OutputFormats> outputFormats = (Set<OutputFormats>) commandMap.get(OUTPUT_FORMATS);
    final StringList countries = StringList.split((String) commandMap.get(COUNTRIES), CommonConstants.COMMA);
    final MapRouletteConfiguration mapRouletteConfiguration = (MapRouletteConfiguration) commandMap.get(MAP_ROULETTE);
    final Configuration checksConfiguration = ConfigurationResolver.loadConfiguration(commandMap, CONFIGURATION_FILES, CONFIGURATION_JSON);
    final boolean saveIntermediateAtlas = (Boolean) commandMap.get(PBF_SAVE_INTERMEDIATE_ATLAS);
    @SuppressWarnings("unchecked") final Rectangle pbfBoundary = ((Optional<Rectangle>) commandMap.getOption(PBF_BOUNDING_BOX)).orElse(Rectangle.MAXIMUM);
    final boolean compressOutput = Boolean.valueOf((String) commandMap.get(SparkJob.COMPRESS_OUTPUT));
    final Map<String, String> sparkContext = configurationMap();
    final CheckResourceLoader checkLoader = new CheckResourceLoader(checksConfiguration);
    // check configuration and country list
    final Set<BaseCheck> preOverriddenChecks = checkLoader.loadChecks();
    if (!isValidInput(countries, preOverriddenChecks)) {
        logger.error("No countries supplied or checks enabled, exiting!");
        return;
    }
    // Read priority countries from the configuration
    final List<String> priorityCountries = checksConfiguration.get("priority.countries", Collections.EMPTY_LIST).value();
    // Create a list of Country to Check tuples
    // Add priority countries first if they are supplied by parameter
    final List<Tuple2<String, Set<BaseCheck>>> countryCheckTuples = new ArrayList<>();
    countries.stream().filter(priorityCountries::contains).forEach(country -> countryCheckTuples.add(new Tuple2<>(country, checkLoader.loadChecksForCountry(country))));
    // Then add the rest of the countries
    countries.stream().filter(country -> !priorityCountries.contains(country)).forEach(country -> countryCheckTuples.add(new Tuple2<>(country, checkLoader.loadChecksForCountry(country))));
    // Log countries and integrity
    logger.info("Initialized countries: {}", countryCheckTuples.stream().map(tuple -> tuple._1).collect(Collectors.joining(",")));
    logger.info("Initialized checks: {}", preOverriddenChecks.stream().map(BaseCheck::getCheckName).collect(Collectors.joining(",")));
    // Parallelize on the countries
    final JavaPairRDD<String, Set<BaseCheck>> countryCheckRDD = getContext().parallelizePairs(countryCheckTuples, countryCheckTuples.size());
    // Set target and temporary folders
    final String targetOutputFolder = SparkFileHelper.parentPath(output);
    final String temporaryOutputFolder = SparkFileHelper.combine(targetOutputFolder, SparkFileHelper.TEMPORARY_FOLDER_NAME);
    // Useful file helper to create/delete/name files and directories
    final SparkFileHelper fileHelper = new SparkFileHelper(sparkContext);
    // Atlas Helper to load different types of Atlas data
    final AtlasDataSource atlasLoader = new AtlasDataSource(sparkContext, checksConfiguration, pbfBoundary);
    // Create target folders
    fileHelper.mkdir(SparkFileHelper.combine(targetOutputFolder, OUTPUT_FLAG_FOLDER));
    fileHelper.mkdir(SparkFileHelper.combine(targetOutputFolder, OUTPUT_GEOJSON_FOLDER));
    fileHelper.mkdir(SparkFileHelper.combine(targetOutputFolder, OUTPUT_METRIC_FOLDER));
    // Run the set of flags per country per check. The output will be an RDD pair mapping each
    // country with a set of SparkFilePaths to flags, geojson and metrics generated.
    final JavaPairRDD<String, Set<SparkFilePath>> resultRDD = countryCheckRDD.mapToPair(tuple -> {
        final Time timer = Time.now();
        final String country = tuple._1();
        final Set<BaseCheck> checks = tuple._2();
        logger.info("Initialized checks for {}: {}", country, checks.stream().map(BaseCheck::getCheckName).collect(Collectors.joining(",")));
        final Set<SparkFilePath> resultingFiles = new HashSet<>();
        final SparkFilePath flagOutput;
        if (outputFormats.contains(OutputFormats.FLAGS)) {
            // Initialize flag output processor
            flagOutput = initializeOutput(OUTPUT_FLAG_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            EventService.get(country).register(new CheckFlagFileProcessor(fileHelper, flagOutput.getTemporaryPath()).withCompression(compressOutput));
        } else {
            flagOutput = null;
        }
        final SparkFilePath geoJsonOutput;
        if (outputFormats.contains(OutputFormats.GEOJSON)) {
            // Initialize geojson output processor
            geoJsonOutput = initializeOutput(OUTPUT_GEOJSON_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            EventService.get(country).register(new CheckFlagGeoJsonProcessor(fileHelper, geoJsonOutput.getTemporaryPath()).withCompression(compressOutput));
        } else {
            geoJsonOutput = null;
        }
        final SparkFilePath metricOutput;
        if (outputFormats.contains(OutputFormats.METRICS)) {
            // Initialize metric output processor
            metricOutput = initializeOutput(OUTPUT_METRIC_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            EventService.get(country).register(new MetricFileGenerator(METRICS_FILENAME, fileHelper, metricOutput.getTemporaryPath()));
        } else {
            metricOutput = null;
        }
        final Consumer<Atlas> intermediateAtlasHandler;
        if (saveIntermediateAtlas) {
            final SparkFilePath atlasOutput = initializeOutput(OUTPUT_ATLAS_FOLDER, TaskContext.get(), country, temporaryOutputFolder, targetOutputFolder);
            intermediateAtlasHandler = atlas -> {
                writeAtlas(atlas, country, atlasOutput, fileHelper);
                resultingFiles.add(atlasOutput);
            };
        } else {
            intermediateAtlasHandler = atlas -> {
            // no-op
            };
        }
        try {
            final Atlas atlas = atlasLoader.load(input, country, intermediateAtlasHandler);
            if (atlas == null) {
                logger.error("Could not find {} Atlas files. Skipping country!", country);
            } else {
                executeChecks(country, atlas, checks, mapRouletteConfiguration);
                // Add output folders for handling later
                Stream.of(flagOutput, metricOutput, geoJsonOutput).filter(Objects::nonNull).forEach(resultingFiles::add);
            }
            EventService.get(country).complete();
            return new Tuple2<>(country, resultingFiles);
        } catch (final CoreException e) {
            logger.error("Exception running integrity checks on {}", country, e);
        }
        logger.trace("Integrity checks took {} ms to execute for {}.", timer.elapsedSince().asMilliseconds(), country);
        return new Tuple2<>(IGNORED_KEY, null);
    }).filter(tuple -> !tuple._1().equals(IGNORED_KEY));
    // Commit results
    resultRDD.foreach(countryPathPair -> {
        final String country = countryPathPair._1();
        final Set<SparkFilePath> paths = countryPathPair._2();
        logger.info("[{}] Committing outputs: {}", country, paths);
        paths.forEach(fileHelper::commit);
    });
    try {
        // Clean up
        logger.info("Deleting {}.", temporaryOutputFolder);
        fileHelper.deleteDirectory(temporaryOutputFolder);
    } catch (final Exception e) {
        logger.warn("Clean up failed!", e);
    }
}

Example 2 with Atlas

use of org.openstreetmap.atlas.geography.atlas.Atlas in project atlas-checks by osmlab.

the class IntegrityCheckSparkJob method executeChecks.

/**
 * Executes all {@link BaseCheck}s on the given {@link Atlas}. Each check runs in a separate
 * thread. The checks go over all {@link AtlasEntity}s and {@link Relation}s.
 * {@link ComplexEntity}s can be processed by using the appropriate {@link Finder} and adding
 * them to the {@link Iterable} of objects.
 *
 * @param atlas
 *            the {@link Atlas} on which the checks will be run
 * @param checksToRun
 *            the set of {@link BaseCheck}s to execute
 * @param configuration
 *            {@link MapRouletteConfiguration} to create a new {@link MapRouletteClient}s
 */
@SuppressWarnings("rawtypes")
private static void executeChecks(final String country, final Atlas atlas, final Set<BaseCheck> checksToRun, final MapRouletteConfiguration configuration) {
    final Pool checkExecutionPool = new Pool(checksToRun.size(), "Check execution pool", POOL_DURATION_BEFORE_KILL);
    checksToRun.stream().filter(check -> check.validCheckForCountry(country)).forEach(check -> checkExecutionPool.queue(new RunnableCheck(country, check, new MultiIterable<>(atlas.items(), atlas.relations(), findComplexEntities(check, atlas)), MapRouletteClient.instance(configuration))));
    checkExecutionPool.close();
}

Example 3 with Atlas

use of org.openstreetmap.atlas.geography.atlas.Atlas in project atlas-checks by osmlab.

the class DuplicateNodeCheckTest method testCheck.

@Test
public void testCheck() {
    final Rectangle bounds = Rectangle.TEST_RECTANGLE;
    final Set<Location> nodes = new HashSet<>();
    while (nodes.size() < 10) {
        nodes.add(Location.random(bounds));
    }
    final PackedAtlasBuilder builder = new PackedAtlasBuilder();
    int identifier = 0;
    for (final Location node : nodes) {
        // for first node make it a duplicate
        if (identifier == 0) {
            builder.addNode(identifier++, node, RandomTagsSupplier.randomTags(5));
        }
        builder.addNode(identifier++, node, RandomTagsSupplier.randomTags(5));
    }
    final Iterator<Location> locationIterator = nodes.iterator();
    final Location start = locationIterator.next();
    identifier = 0;
    while (locationIterator.hasNext()) {
        if (identifier == 0) {
            builder.addEdge(identifier++, new Segment(start, locationIterator.next()), RandomTagsSupplier.randomTags(5));
        }
        builder.addEdge(identifier++, new Segment(start, locationIterator.next()), RandomTagsSupplier.randomTags(5));
    }
    final Atlas atlas = builder.get();
    final Iterable<CheckFlag> flags = new DuplicateNodeCheck(ConfigurationResolver.emptyConfiguration()).flags(atlas);
    Assert.assertEquals(1, Iterables.size(flags));
}

Example 4 with Atlas

use of org.openstreetmap.atlas.geography.atlas.Atlas in project atlas-checks by osmlab.

the class EdgeCrossingEdgeCheck method flag.

@Override
protected Optional<CheckFlag> flag(final AtlasObject object) {
    // Prepare the edge being tested for checks
    final Edge edge = (Edge) object;
    final PolyLine edgeAsPolyLine = edge.asPolyLine();
    final Rectangle edgeBounds = edge.bounds();
    final Optional<Long> edgeLayer = LayerTag.getTaggedOrImpliedValue(object, 0L);
    // Retrieve crossing edges
    final Atlas atlas = object.getAtlas();
    final Iterable<Edge> crossingEdges = atlas.edgesIntersecting(edgeBounds, // filter out the same edge, non-valid crossing edges and already flagged ones
    crossingEdge -> edge.getIdentifier() != crossingEdge.getIdentifier() && isValidCrossingEdge(crossingEdge) && !this.isFlagged(generateAtlasObjectPairIdentifier(edge, crossingEdge)));
    List<Edge> invalidEdges = null;
    // each edge will be marked explicitly.
    for (final Edge crossingEdge : crossingEdges) {
        final PolyLine crossingEdgeAsPolyLine = crossingEdge.asPolyLine();
        final Optional<Long> crossingEdgeLayer = LayerTag.getTaggedOrImpliedValue(crossingEdge, 0L);
        final Set<Location> intersections = edgeAsPolyLine.intersections(crossingEdgeAsPolyLine);
        // Check whether crossing edge can actually cross
        for (final Location intersection : intersections) {
            // Add this set to flagged pairs to skip it next time
            this.markAsFlagged(generateAtlasObjectPairIdentifier(edge, crossingEdge));
            // Check if crossing is valid or not
            if (canCross(edgeAsPolyLine, edgeLayer, crossingEdgeAsPolyLine, crossingEdgeLayer, intersection)) {
                continue;
            }
            if (invalidEdges == null) {
                // Normally we expect 1 or 2 edges in the list
                invalidEdges = new LinkedList<>();
            }
            invalidEdges.add(crossingEdge);
        }
    }
    if (invalidEdges != null) {
        final CheckFlag newFlag = new CheckFlag(getTaskIdentifier(object));
        newFlag.addObject(object);
        newFlag.addInstruction(this.getLocalizedInstruction(0, object.getOsmIdentifier()));
        invalidEdges.forEach(invalidEdge -> {
            newFlag.addObject(invalidEdge);
            newFlag.addInstruction(this.getLocalizedInstruction(1, invalidEdge.getOsmIdentifier()));
        });
        return Optional.of(newFlag);
    }
    return Optional.empty();
}

Example 5 with Atlas

use of org.openstreetmap.atlas.geography.atlas.Atlas in project atlas-checks by osmlab.

the class TagTest method testConfiguration.

/**
 * Private function used by all test cases to easily test the configuration instead of rewriting
 * the code constantly.
 *
 * @param config
 *            The {@link Configuration} is string form that is being test.
 * @param numberOfFlags
 *            The number of expected flags to be processed based on the provided configuration
 */
private void testConfiguration(final String config, final int numberOfFlags) {
    final Atlas atlas = setup.getAtlas();
    final Configuration configuration = ConfigurationResolver.inlineConfiguration(config);
    final List<CheckFlag> flags = Iterables.asList(new BaseTestCheck(configuration).flags(atlas));
    // will return the number of flags for the number of objects because no restrictions
    Assert.assertEquals(numberOfFlags, flags.size());
}