Examples with CsvDatumExportOutputFormatService net.solarnetwork.central.datum.export.standard.CsvDatumExportOutputFormatService used on opensource projects

Example 1 with CsvDatumExportOutputFormatService

use of net.solarnetwork.central.datum.export.standard.CsvDatumExportOutputFormatService in project solarnetwork-central by SolarNetwork.

the class DatumExportStandardServiceConfig method csvDatumExportOutputFormatService.

@Bean
public DatumExportOutputFormatService csvDatumExportOutputFormatService() {
    CsvDatumExportOutputFormatService service = new CsvDatumExportOutputFormatService();
    service.setTemporaryPath(temporaryDir);
    ResourceBundleMessageSource msgSource = new ResourceBundleMessageSource();
    msgSource.setBasenames(CsvDatumExportOutputFormatService.class.getName(), OutputConfiguration.class.getName());
    service.setMessageSource(msgSource);
    return service;
}

Example 2 with CsvDatumExportOutputFormatService

use of net.solarnetwork.central.datum.export.standard.CsvDatumExportOutputFormatService in project solarnetwork-central by SolarNetwork.

the class S3DatumExportDestinationServiceTests method export.

@Test
public void export() throws IOException {
    // given
    AmazonS3 client = getS3Client();
    cleanS3Folder(client);
    S3DatumExportDestinationService service = new S3DatumExportDestinationService();
    Instant ts = LocalDateTime.of(2018, 4, 11, 11, 50).atZone(ZoneId.of("Pacific/Auckland")).toInstant();
    BasicConfiguration config = new BasicConfiguration();
    BasicDestinationConfiguration destConfig = new BasicDestinationConfiguration();
    destConfig.setServiceIdentifier(service.getId());
    Map<String, Object> destProps = new HashMap<>();
    TEST_PROPS.forEach((k, v) -> destProps.put(k.toString(), v));
    destConfig.setServiceProps(destProps);
    config.setDestinationConfiguration(destConfig);
    Map<String, Object> runtimeProps = config.createRuntimeProperties(ts, null, new CsvDatumExportOutputFormatService());
    DatumExportResource rsrc = getTestResource();
    // when
    List<Double> progress = new ArrayList<>(8);
    service.export(config, Collections.singleton(rsrc), runtimeProps, new ProgressListener<DatumExportService>() {

        @Override
        public void progressChanged(DatumExportService context, double amountComplete) {
            assertThat("Context is service", context, sameInstance(service));
            progress.add(amountComplete);
        }
    });
    // then
    assertThat("Progress was made", progress, not(hasSize(0)));
    assertThat("Progress complete", progress.get(progress.size() - 1), equalTo((Double) 1.0));
    // now list our folder to verify expected result
    ListObjectsV2Result listing = listFolder(client);
    Set<String> keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toSet());
    String keyPrefix = getObjectKeyPrefix();
    assertThat(keys, containsInAnyOrder(keyPrefix + "/data-export-2018-04-10.csv"));
    String exportContent = getObjectAsString(client, listing.getObjectSummaries().get(0).getKey());
    assertThat("Exported content", exportContent, equalTo(FileCopyUtils.copyToString(new InputStreamReader(getClass().getResourceAsStream("test-datum-export-01.txt"), "UTF-8"))));
}

Example 3 with CsvDatumExportOutputFormatService

use of net.solarnetwork.central.datum.export.standard.CsvDatumExportOutputFormatService in project solarnetwork-central by SolarNetwork.

the class CsvDatumExportOutputFormatServiceTests method exportCompressedGzip.

@Test
public void exportCompressedGzip() throws IOException {
    // given
    CsvDatumExportOutputFormatService service = new CsvDatumExportOutputFormatService();
    BasicOutputConfiguration config = new BasicOutputConfiguration();
    config.setCompressionType(OutputCompressionType.GZIP);
    List<GeneralNodeDatumFilterMatch> data = new ArrayList<>(100);
    Instant start = LocalDateTime.of(2018, 4, 23, 11, 19).atZone(ZoneOffset.UTC).toInstant();
    for (int i = 0; i < 100; i++) {
        GeneralNodeDatumMatch d1 = new GeneralNodeDatumMatch();
        d1.setCreated(start.plus(i, ChronoUnit.MINUTES));
        d1.setNodeId(TEST_NODE_ID);
        d1.setSourceId(TEST_SOURCE_ID);
        d1.setLocalDateTime(d1.getCreated().atZone(ZoneOffset.UTC).toLocalDateTime());
        d1.setSampleJson("{\"i\":{\"watts\":123.456}}");
        data.add(d1);
    }
    List<Double> progress = new ArrayList<>(4);
    // when
    Iterable<DatumExportResource> results = null;
    try (DatumExportOutputFormatService.ExportContext context = service.createExportContext(config)) {
        assertThat("Context created", context, notNullValue());
        context.start(1);
        context.appendDatumMatch(data, new ProgressListener<DatumExportService>() {

            @Override
            public void progressChanged(DatumExportService ctx, double amountComplete) {
                assertThat("Same context", ctx, sameInstance(service));
                progress.add(amountComplete);
            }
        });
        results = context.finish();
    }
    // then
    assertThat("Result created", results, notNullValue());
    List<DatumExportResource> resultList = StreamSupport.stream(results.spliterator(), false).collect(Collectors.toList());
    assertThat(resultList, hasSize(1));
    assertThat("Progress provided", progress, hasSize(100));
    DatumExportResource r = resultList.get(0);
    assertThat(r, Matchers.instanceOf(BasicDatumExportResource.class));
    File tempFile = ((BasicDatumExportResource) r).getDelegate().getFile();
    assertThat("Temp file exists", tempFile.exists(), equalTo(true));
    assertThat("Temp file extension", tempFile.getName(), endsWith(".csv.gz"));
    assertThat("Content type", r.getContentType(), equalTo(OutputCompressionType.GZIP.getContentType()));
    String csv = FileCopyUtils.copyToString(new InputStreamReader(new GZIPInputStream(r.getInputStream()), "UTF-8"));
    assertThat("Temp file deleted", tempFile.exists(), equalTo(false));
    StringBuilder buf = new StringBuilder("created,nodeId,sourceId,localDate,localTime,watts\r\n");
    DateTimeFormatter tsFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'hh:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC);
    DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC);
    DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("hh:mm:ss.SSS").withZone(ZoneOffset.UTC);
    for (int i = 0; i < 100; i++) {
        Instant date = start.plus(i, ChronoUnit.MINUTES);
        buf.append(String.format("%s,-1,test.source,%s,%s,123.456\r\n", tsFormatter.format(date), dateFormatter.format(date), timeFormatter.format(date)));
    }
    assertThat("Generated CSV", csv, equalTo(buf.toString()));
}

Example 4 with CsvDatumExportOutputFormatService

use of net.solarnetwork.central.datum.export.standard.CsvDatumExportOutputFormatService in project solarnetwork-central by SolarNetwork.

the class CsvDatumExportOutputFormatServiceTests method exportCompressedGzipNoHeader.

@Test
public void exportCompressedGzipNoHeader() throws IOException {
    // given
    CsvDatumExportOutputFormatService service = new CsvDatumExportOutputFormatService();
    BasicOutputConfiguration config = new BasicOutputConfiguration();
    config.setCompressionType(OutputCompressionType.GZIP);
    config.setServiceProps(Collections.singletonMap("includeHeader", "false"));
    List<GeneralNodeDatumFilterMatch> data = new ArrayList<>(100);
    Instant start = LocalDateTime.of(2018, 4, 23, 11, 19).atZone(ZoneOffset.UTC).toInstant();
    for (int i = 0; i < 100; i++) {
        GeneralNodeDatumMatch d1 = new GeneralNodeDatumMatch();
        d1.setCreated(start.plus(i, ChronoUnit.MINUTES));
        d1.setNodeId(TEST_NODE_ID);
        d1.setSourceId(TEST_SOURCE_ID);
        d1.setLocalDateTime(d1.getCreated().atZone(ZoneOffset.UTC).toLocalDateTime());
        d1.setSampleJson("{\"i\":{\"watts\":123.456}}");
        data.add(d1);
    }
    List<Double> progress = new ArrayList<>(4);
    // when
    Iterable<DatumExportResource> results = null;
    try (DatumExportOutputFormatService.ExportContext context = service.createExportContext(config)) {
        assertThat("Context created", context, notNullValue());
        context.start(1);
        context.appendDatumMatch(data, new ProgressListener<DatumExportService>() {

            @Override
            public void progressChanged(DatumExportService ctx, double amountComplete) {
                assertThat("Same context", ctx, sameInstance(service));
                progress.add(amountComplete);
            }
        });
        results = context.finish();
    }
    // then
    assertThat("Result created", results, notNullValue());
    List<DatumExportResource> resultList = StreamSupport.stream(results.spliterator(), false).collect(Collectors.toList());
    assertThat(resultList, hasSize(1));
    assertThat("Progress provided", progress, hasSize(100));
    DatumExportResource r = resultList.get(0);
    assertThat(r, Matchers.instanceOf(BasicDatumExportResource.class));
    File tempFile = ((BasicDatumExportResource) r).getDelegate().getFile();
    assertThat("Temp file exists", tempFile.exists(), equalTo(true));
    assertThat("Temp file extension", tempFile.getName(), endsWith(".csv.gz"));
    assertThat("Content type", r.getContentType(), equalTo(OutputCompressionType.GZIP.getContentType()));
    String csv = FileCopyUtils.copyToString(new InputStreamReader(new GZIPInputStream(r.getInputStream()), "UTF-8"));
    assertThat("Temp file deleted", tempFile.exists(), equalTo(false));
    StringBuilder buf = new StringBuilder();
    DateTimeFormatter tsFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'hh:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC);
    DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC);
    DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("hh:mm:ss.SSS").withZone(ZoneOffset.UTC);
    for (int i = 0; i < 100; i++) {
        Instant date = start.plus(i, ChronoUnit.MINUTES);
        buf.append(String.format("%s,-1,test.source,%s,%s,123.456\r\n", tsFormatter.format(date), dateFormatter.format(date), timeFormatter.format(date)));
    }
    assertThat("Generated CSV", csv, equalTo(buf.toString()));
}

Example 5 with CsvDatumExportOutputFormatService

use of net.solarnetwork.central.datum.export.standard.CsvDatumExportOutputFormatService in project solarnetwork-central by SolarNetwork.

the class CsvDatumExportOutputFormatServiceTests method export.

@Test
public void export() throws IOException {
    // given
    CsvDatumExportOutputFormatService service = new CsvDatumExportOutputFormatService();
    BasicOutputConfiguration config = new BasicOutputConfiguration();
    config.setCompressionType(OutputCompressionType.None);
    GeneralNodeDatumMatch d1 = new GeneralNodeDatumMatch();
    d1.setCreated(LocalDateTime.of(2018, 4, 23, 11, 19).atZone(ZoneOffset.UTC).toInstant());
    d1.setNodeId(TEST_NODE_ID);
    d1.setSourceId(TEST_SOURCE_ID);
    d1.setLocalDateTime(d1.getCreated().atZone(ZoneOffset.UTC).toLocalDateTime());
    d1.setPosted(d1.getCreated().plus(1, ChronoUnit.MINUTES));
    d1.setSampleJson("{\"i\":{\"watts\":123.456}}");
    List<GeneralNodeDatumFilterMatch> data = Arrays.asList(d1);
    List<Double> progress = new ArrayList<>(4);
    // when
    Iterable<DatumExportResource> results = null;
    try (DatumExportOutputFormatService.ExportContext context = service.createExportContext(config)) {
        assertThat("Context created", context, notNullValue());
        context.start(1);
        context.appendDatumMatch(data, new ProgressListener<DatumExportService>() {

            @Override
            public void progressChanged(DatumExportService ctx, double amountComplete) {
                assertThat("Same context", ctx, sameInstance(service));
                progress.add(amountComplete);
            }
        });
        results = context.finish();
    }
    // then
    assertThat("Result created", results, notNullValue());
    assertThat("Progress provided", progress, hasSize(1));
    List<DatumExportResource> resultList = StreamSupport.stream(results.spliterator(), false).collect(Collectors.toList());
    assertThat(resultList, hasSize(1));
    DatumExportResource r = resultList.get(0);
    assertThat(r, Matchers.instanceOf(BasicDatumExportResource.class));
    File tempFile = ((BasicDatumExportResource) r).getDelegate().getFile();
    assertThat("Temp file exists", tempFile.exists(), equalTo(true));
    assertThat("Temp file extension", tempFile.getName(), endsWith(".csv"));
    assertThat("Content type", r.getContentType(), equalTo(service.getExportContentType()));
    String csv = FileCopyUtils.copyToString(new InputStreamReader(r.getInputStream(), "UTF-8"));
    assertThat("Temp file deleted", tempFile.exists(), equalTo(false));
    assertThat("Generated CSV", csv, equalTo("created,nodeId,sourceId,localDate,localTime,watts\r\n" + "2018-04-23T11:19:00.000Z,-1,test.source,2018-04-23,11:19:00.000,123.456\r\n"));
}