Examples with CountryBoundaryMap org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap used on opensource projects

Example 21 with CountryBoundaryMap

use of org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap in project atlas by osmlab.

the class AnyToGeoJsonCommand method executeBoundaryContext.

private int executeBoundaryContext() {
    Set<String> countries = new HashSet<>();
    if (this.optionAndArgumentDelegate.hasOption(COUNTRIES_OPTION_LONG)) {
        countries = this.optionAndArgumentDelegate.getOptionArgument(COUNTRIES_OPTION_LONG, this::parseCommaSeparatedCountries).orElse(new HashSet<>());
    }
    final boolean usePolygons = this.optionAndArgumentDelegate.hasOption(POLYGONS_OPTION_LONG);
    Set<String> countriesDenyList = new HashSet<>();
    if (this.optionAndArgumentDelegate.hasOption(COUNTRIES_DENY_LIST_OPTION_LONG)) {
        countriesDenyList = this.optionAndArgumentDelegate.getOptionArgument(COUNTRIES_DENY_LIST_OPTION_LONG, this::parseCommaSeparatedCountries).orElse(new HashSet<>());
    }
    if (this.optionAndArgumentDelegate.hasVerboseOption()) {
        this.outputDelegate.printlnCommandMessage("reading CountryBoundaryMap from file...");
    }
    final CountryBoundaryMap map = CountryBoundaryMap.fromPlainText(new File(this.optionAndArgumentDelegate.getOptionArgument(COUNTRY_BOUNDARY_OPTION_LONG).orElseThrow(AtlasShellToolsException::new)));
    final String boundaryJson;
    if (countries.isEmpty()) {
        boundaryJson = new CountryBoundaryMapGeoJsonConverter().prettyPrint(true).withCountryDenyList(countriesDenyList).usePolygons(usePolygons).convertToString(map);
    } else {
        boundaryJson = new CountryBoundaryMapGeoJsonConverter().withCountryAllowList(countries).withCountryDenyList(countriesDenyList).prettyPrint(true).usePolygons(usePolygons).convertToString(map);
    }
    if (this.optionAndArgumentDelegate.hasVerboseOption()) {
        this.outputDelegate.printlnCommandMessage("converting boundary file to GeoJSON...");
    }
    if (this.optionAndArgumentDelegate.hasVerboseOption()) {
        this.outputDelegate.printlnCommandMessage("writing the boundary file...");
    }
    final Path concatenatedPath = Paths.get(getOutputPath().toAbsolutePath().toString(), BOUNDARY_FILE);
    new File(concatenatedPath.toAbsolutePath().toString()).writeAndClose(boundaryJson);
    return 0;
}

Example 22 with CountryBoundaryMap

use of org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap in project atlas-generator by osmlab.

the class ShardedSparkJob method start.

@Override
public void start(final CommandMap command) {
    final StringList countries = (StringList) command.get(COUNTRIES);
    final String shardingType = (String) command.get(SHARDING_TYPE);
    final Sharding sharding = AtlasSharding.forString(shardingType, configuration());
    final String countryShapes = (String) command.get(COUNTRY_SHAPES);
    logger.info("Reading country boundaries from {}", countryShapes);
    final CountryBoundaryMap worldBoundaries = new CountryBoundaryMapArchiver().read(resource(countryShapes));
    logger.info("Done Reading {} country boundaries from {}", worldBoundaries.size(), countryShapes);
    // The code below is as parallel as there are countries...
    final JavaRDD<String> countriesRDD = getContext().parallelize(Iterables.asList(countries), countries.size());
    // Stage 0 (parallelize on countries)
    final JavaPairRDD<String, Shard> preCountryShardsRDD = countriesRDD.flatMapToPair(countryToShards(worldBoundaries, sharding));
    // Collect The country to shard map
    final List<Tuple2<String, Shard>> countryShards = preCountryShardsRDD.collect();
    final MultiMap<String, Shard> countryToShardMap = new MultiMap<>();
    countryShards.forEach(tuple -> countryToShardMap.add(tuple._1(), tuple._2()));
    // Call the rest!
    start2(command, sharding, countryToShardMap);
}

Example 23 with CountryBoundaryMap

use of org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap in project atlas-generator by osmlab.

the class DynamicRelationSlicingIntegrationTest method testRelationSlicing.

/**
 * This integration test country slices a water relation that spans four shards, and crosses a
 * country boundary. The relation is country sliced twice, once from the ABC perspective and
 * once from DEF. The resulting closed relations then have their areas validated.
 */
@Ignore
@Test
public void testRelationSlicing() {
    // parameter setup
    final Sharding sharding = new DynamicTileSharding(new InputStreamResource(() -> DynamicRelationSlicingIntegrationTest.class.getResourceAsStream("simpleSharding.txt")));
    final CountryBoundaryMap boundaryMap = CountryBoundaryMap.fromPlainText(new InputStreamResource(() -> DynamicRelationSlicingIntegrationTest.class.getResourceAsStream("simpleBoundaryFile.txt")));
    final Map<String, String> sparkOptions = new HashMap<>();
    sparkOptions.put("spark.executor.memory", "512m");
    sparkOptions.put("spark.executor.memory", "512m");
    sparkOptions.put("spark.driver.memory", "512m");
    ResourceFileSystem.simpleconfiguration().forEach((key, value) -> sparkOptions.put(key, value));
    final HadoopAtlasFileCache lineSlicedAtlasCache = new HadoopAtlasFileCache("resource://test/atlas/", sparkOptions);
    final CountryShard initialShardABC = CountryShard.forName("ABC_1-0-0");
    final CountryShard initialShardDEF = CountryShard.forName("DEF_1-1-1");
    final AtlasResourceLoader loader = new AtlasResourceLoader();
    final Atlas rawAtlasABC = loader.load(lineSlicedAtlasCache.get(initialShardABC.getCountry(), initialShardABC.getShard()).get());
    final Atlas rawAtlasDEF = loader.load(lineSlicedAtlasCache.get(initialShardDEF.getCountry(), initialShardDEF.getShard()).get());
    final Function<Shard, Optional<Atlas>> atlasFetcherABC = AtlasGeneratorHelper.atlasFetcher(lineSlicedAtlasCache, rawAtlasABC, boundaryMap, "ABC", initialShardDEF.getShard());
    final Function<Shard, Optional<Atlas>> atlasFetcherDEF = AtlasGeneratorHelper.atlasFetcher(lineSlicedAtlasCache, rawAtlasDEF, boundaryMap, "DEF", initialShardDEF.getShard());
    // the operation we are testing
    final Atlas relationSlicedAtlasDEF = new RawAtlasSlicer(AtlasLoadingOption.createOptionWithAllEnabled(boundaryMap).setCountryCode("DEF"), initialShardDEF.getShard(), sharding, atlasFetcherDEF).slice();
    final Atlas relationSlicedAtlasABC = new RawAtlasSlicer(AtlasLoadingOption.createOptionWithAllEnabled(boundaryMap).setCountryCode("ABC"), initialShardABC.getShard(), sharding, atlasFetcherABC).slice();
    // Sum the areas and check it is the size expected
    double totalAreaDEF = 0;
    for (final Line line : relationSlicedAtlasDEF.lines()) {
        if (line.isClosed()) {
            final Polygon polygon = new Polygon(line.asPolyLine());
            totalAreaDEF += polygon.surface().asDm7Squared();
        }
    }
    Assert.assertEquals(5.9870426059188685E17, totalAreaDEF, 0);
    double totalAreaABC = 0;
    for (final Line line : relationSlicedAtlasABC.lines()) {
        if (line.isClosed()) {
            final Polygon polygon = new Polygon(line.asPolyLine());
            totalAreaABC += polygon.surface().asDm7Squared();
        }
    }
    Assert.assertEquals(8.9912430112087885E17, totalAreaABC, 0);
}

Example 24 with CountryBoundaryMap

use of org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap in project atlas-generator by osmlab.

the class AtlasMissingShardVerifier method onRun.

@Override
protected int onRun(final CommandMap command) {
    final CountryBoundaryMap boundaries = (CountryBoundaryMap) command.get(BOUNDARIES);
    final File missingShardFile = (File) command.get(MISSING_SHARDS);
    final File output = (File) command.get(OUTPUT);
    if (!missingShardFile.exists() || missingShardFile.all().equals("")) {
        logger.info("No missing shards to check!");
        return 0;
    }
    final Set<CountryShard> missingCountryShards = missingShardFile.linesList().stream().map(CountryShard::forName).collect(Collectors.toSet());
    final String wayFilterPath = (String) command.get(WAY_FILTER);
    final ConfiguredTaggableFilter wayFilter;
    if (wayFilterPath != null) {
        wayFilter = AtlasGeneratorParameters.getTaggableFilterFrom(wayFilterPath, Maps.hashMap());
    } else {
        wayFilter = new ConfiguredTaggableFilter(new StandardConfiguration(new InputStreamResource(() -> AtlasMissingShardVerifier.class.getResourceAsStream(FILTER_CONFIG))));
    }
    final String server = (String) command.get(OVERPASS_SERVER);
    final StringList proxySettings = (StringList) command.get(PROXY_SETTINGS);
    final Integer numRetryQueries = (Integer) command.get(RETRY_QUERY_COUNT);
    HttpHost proxy = null;
    if (proxySettings != null) {
        try {
            proxy = createProxy(proxySettings);
        } catch (final Exception e) {
            logger.error("Proxy settings not correctly set", e);
            return -1;
        }
    }
    final Time fullStart = Time.now();
    final int returnCode;
    try {
        returnCode = verifier(boundaries, missingCountryShards, output, server, proxy, wayFilter, numRetryQueries);
    } catch (final Exception e) {
        return -1;
    }
    logger.info("Verification ran in: {}", fullStart.elapsedSince());
    return returnCode;
}

Example 25 with CountryBoundaryMap

use of org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap in project atlas-generator by osmlab.

the class AtlasMissingShardVerifier method initializeCountryBoundaryMap.

private static CountryBoundaryMap initializeCountryBoundaryMap(final String value) {
    final Time start = Time.now();
    logger.info("Loading boundaries");
    final CountryBoundaryMap result = CountryBoundaryMap.fromPlainText(new File(value));
    logger.info("Loaded boundaries in {}", start.elapsedSince());
    return result;
}