Examples with WritableResource org.openstreetmap.atlas.streaming.resource.WritableResource used on opensource projects

Example 1 with WritableResource

use of org.openstreetmap.atlas.streaming.resource.WritableResource in project atlas-generator by osmlab.

the class ShardedAtlasRDDLoaderTest method addTextAtlasResource.

private static void addTextAtlasResource(final String path, final String name) {
    final TextAtlasBuilder builder = new TextAtlasBuilder();
    final PackedAtlas packedAtlas = builder.read(new InputStreamResource(() -> ShardedAtlasRDDLoaderTest.class.getResourceAsStream(name)));
    final WritableResource packedAtlasResource = new ByteArrayResource();
    packedAtlas.save(packedAtlasResource);
    ResourceFileSystem.addResource(path, packedAtlasResource);
}

Example 2 with WritableResource

use of org.openstreetmap.atlas.streaming.resource.WritableResource in project atlas-generator by osmlab.

the class AtlasShardVerifier method onRun.

@Override
protected int onRun(final CommandMap command) {
    final String atlasFolder = getPathFor(command, ATLAS_FOLDER);
    final int depth = (int) command.get(LIST_DEPTH);
    final Pattern pattern = (Pattern) command.get(PATH_FILTER_REGEX);
    @SuppressWarnings("unchecked") final Set<String> countries = (Set<String>) command.get(COUNTRIES);
    logger.debug("Using regex filter \"{}\"", pattern);
    final PathFilter filter = path -> pattern.matcher(path.toString()).matches();
    final Map<String, String> sparkConfiguration = sparkOptions(command);
    final WritableResource output = FileSystemHelper.writableResource(getPathFor(command, OUTPUT), sparkConfiguration);
    final String expectedShardsPath = getPathFor(command, EXPECTED_SHARDS);
    Set<CountryShard> expectedShards;
    if (FileSystemHelper.isFile(expectedShardsPath, sparkConfiguration)) {
        logger.trace("isFile: {}", expectedShardsPath);
        expectedShards = FileSystemHelper.resource(expectedShardsPath, sparkConfiguration).linesList().stream().map(CountryShard::forName).collect(Collectors.toSet());
    } else if (FileSystemHelper.isDirectory(expectedShardsPath, sparkConfiguration)) {
        logger.trace("isDirectory: {}", expectedShardsPath);
        expectedShards = shardsFromFolder(expectedShardsPath, sparkConfiguration, depth, filter);
    } else {
        throw new CoreException("{} does not exist.", expectedShardsPath);
    }
    expectedShards = expectedShards.stream().filter(countryShard -> countries.isEmpty() || countries.contains(countryShard.getCountry())).collect(Collectors.toSet());
    final Set<CountryShard> existingShards = shardsFromFolder(atlasFolder, sparkConfiguration, depth, filter).stream().filter(countryShard -> countries.isEmpty() || countries.contains(countryShard.getCountry())).collect(Collectors.toSet());
    expectedShards.removeAll(existingShards);
    try (SafeBufferedWriter writer = output.writer()) {
        expectedShards.stream().map(CountryShard::getName).forEach(writer::writeLine);
    } catch (final Exception e) {
        throw new CoreException("Verification failed", e);
    }
    return 0;
}

Example 3 with WritableResource

use of org.openstreetmap.atlas.streaming.resource.WritableResource in project atlas-generator by osmlab.

the class SparkFileHelper method copyFile.

private void copyFile(final Resource resource, final String targetPath) {
    final WritableResource output = FileSystemHelper.writableResource(targetPath, this.sparkContext);
    resource.copyTo(output);
}

Example 4 with WritableResource

use of org.openstreetmap.atlas.streaming.resource.WritableResource in project atlas-generator by osmlab.

the class ResourceFileSystem method create.

@Override
public FSDataOutputStream create(final Path hadoopPath, final FsPermission permission, final boolean overwrite, final int bufferSize, final short replication, final long blockSize, final Progressable progress) throws IOException {
    if (STORE.containsKey(hadoopPath.toString())) {
        delete(hadoopPath, false);
    }
    final String name = hadoopPath.toString();
    final WritableResource resource = new ByteArrayResource().withName(name);
    STORE.put(name, resource);
    return new FSDataOutputStream(resource.write(), STATISTICS_INTERNAL);
}

Example 5 with WritableResource

use of org.openstreetmap.atlas.streaming.resource.WritableResource in project atlas-generator by osmlab.

the class AtlasShardVerifierTest method testCountryFiltering.

@Test
public void testCountryFiltering() {
    final WritableResource atlas = new StringResource("blah");
    ResourceFileSystem.addResource(CHECK + "/ABC/ABC_10-11-12.atlas", atlas);
    ResourceFileSystem.addResource(CHECK + "/ABC/ABC_10-11-13.atlas", atlas);
    ResourceFileSystem.addResource(CHECK + "/DEF/DEF_10-11-14.atlas", atlas);
    ResourceFileSystem.addResource(CHECK + "/DEF/DEF_10-11-15.atlas", atlas);
    ResourceFileSystem.addResource(INPUT + "/ABC/ABC_10-11-12.atlas", atlas);
    ResourceFileSystem.addResource(INPUT + "/XYZ/XYZ_10-11-13.atlas", atlas);
    ResourceFileSystem.addResource(INPUT + "/DEF/DEF_10-11-14.atlas", atlas);
    new AtlasShardVerifier().runWithoutQuitting(getArguments());
    Assert.assertEquals("ABC_10-11-13", FileSystemHelper.resource(OUTPUT, ResourceFileSystem.simpleconfiguration()).all());
}