Examples with SparkFileHelper org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileHelper used on opensource projects

Example 6 with SparkFileHelper

use of org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileHelper in project atlas-checks by osmlab.

the class MetricFileGeneratorTest method testFileName.

@Test
public void testFileName() throws IOException {
    final File tempDirectory = this.tempFolder.newFolder();
    final MetricFileGenerator generator = new MetricFileGenerator("some-metrics-log.csv", new SparkFileHelper(FILE_SYSTEM_CONFIG), tempDirectory.getAbsolutePath());
    final String filename = generator.getFilename();
    Assert.assertTrue(filename.matches("^some-metrics-log-\\d+.csv$"));
}

Example 7 with SparkFileHelper

use of org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileHelper in project atlas-checks by osmlab.

the class MetricFileGeneratorTest method processCompleteAndValidate.

private void processCompleteAndValidate(final int eventCount) throws IOException {
    // Generate
    final File tempDirectory = this.tempFolder.newFolder();
    final MetricFileGenerator processor = new MetricFileGenerator("some-file-name.csv", new SparkFileHelper(FILE_SYSTEM_CONFIG), tempDirectory.getAbsolutePath());
    for (int index = 0; index < eventCount; index++) {
        processor.process(SAMPLE_EVENT);
    }
    processor.process(new ShutdownEvent());
    // Validate
    final List<Resource> files = FileSystemHelper.resources(tempDirectory.getAbsolutePath(), FILE_SYSTEM_CONFIG);
    Assert.assertEquals((int) Math.floor(eventCount / (double) BATCH_SIZE) + 1, files.size());
    int actualEventCount = 0;
    for (final Resource file : files) {
        for (final String line : file.lines()) {
            // This is first line
            if (actualEventCount == 0) {
                Assert.assertEquals(MetricEvent.header(), line);
            } else {
                Assert.assertEquals("a-metric-name,60000", line);
            }
            actualEventCount++;
        }
    }
    // Plus 1 is for the header
    Assert.assertEquals(eventCount + 1, actualEventCount);
}