Examples with DataSetMetadata org.talend.dataprep.api.dataset.DataSetMetadata used on opensource projects

Example 26 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class DataSetJSONTest method testRoundTrip.

@Test
public void testRoundTrip() throws Exception {
    DataSet dataSet = from(DataSetJSONTest.class.getResourceAsStream("test3.json"));
    final DataSetMetadata metadata = dataSet.getMetadata();
    metadata.getContent().addParameter(CSVFormatFamily.SEPARATOR_PARAMETER, ",");
    metadata.getContent().setFormatFamilyId(new CSVFormatFamily().getBeanId());
    assertNotNull(metadata);
    StringWriter writer = new StringWriter();
    to(dataSet, writer);
    assertThat(writer.toString(), sameJSONAsFile(DataSetJSONTest.class.getResourceAsStream("test3.json")));
}

Example 27 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class AggregationService method aggregate.

/**
 * Process an aggregation.
 *
 * @param parameters the aggregation parameters.
 * @param dataset the dataset input.
 * @return the aggregation result.
 */
public AggregationResult aggregate(AggregationParameters parameters, DataSet dataset) {
    // check the parameters
    if (parameters.getOperations().isEmpty() || parameters.getGroupBy().isEmpty()) {
        throw new TDPException(CommonErrorCodes.BAD_AGGREGATION_PARAMETERS);
    }
    AggregationResult result = new AggregationResult(parameters.getOperations().get(0).getOperator());
    // get the aggregator
    Aggregator aggregator = factory.get(parameters);
    // Build optional filter
    final DataSetMetadata metadata = dataset.getMetadata();
    final RowMetadata rowMetadata = metadata != null ? metadata.getRowMetadata() : new RowMetadata();
    final Predicate<DataSetRow> filter = filterService.build(parameters.getFilter(), rowMetadata);
    // process the dataset
    dataset.getRecords().filter(filter).forEach(row -> aggregator.accept(row, result));
    // Normalize result (perform clean / optimization now that all input was processed).
    aggregator.normalize(result);
    return result;
}

Example 28 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class DataSetServiceTest method datePattern.

@Test
public void datePattern() throws Exception {
    int before = dataSetMetadataRepository.size();
    String dataSetId = given().body(IOUtils.toString(this.getClass().getResourceAsStream("../date_time_pattern.csv"), UTF_8)).queryParam(CONTENT_TYPE, "text/csv").when().post("/datasets").asString();
    int after = dataSetMetadataRepository.size();
    assertThat(after - before, is(1));
    assertQueueMessages(dataSetId);
    final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId);
    assertNotNull(dataSetMetadata);
    final ColumnMetadata column = dataSetMetadata.getRowMetadata().getById("0001");
    assertThat(column.getType(), is("date"));
    assertThat(column.getDomain(), is(""));
    final Statistics statistics = mapper.readerFor(Statistics.class).readValue(this.getClass().getResourceAsStream("../date_time_pattern_expected.json"));
    assertThat(column.getStatistics(), CoreMatchers.equalTo(statistics));
}

Example 29 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class DataSetServiceTest method previewNonDraft.

@Test
public void previewNonDraft() throws Exception {
    // Create a data set
    String dataSetId = given().body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)).queryParam("Content-Type", "text/csv").when().post("/datasets").asString();
    final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId);
    assertThat(dataSetMetadata, notNullValue());
    // Ensure it is no draft
    dataSetMetadata.setDraft(false);
    dataSetMetadataRepository.save(dataSetMetadata);
    // Should receive a 301 that redirects to the GET data set content operation
    // 
    given().redirects().follow(false).contentType(JSON).get("/datasets/{id}/preview", dataSetId).then().statusCode(HttpStatus.MOVED_PERMANENTLY.value());
    // Should receive a 200 if code follows redirection
    // 
    given().redirects().follow(true).contentType(JSON).get("/datasets/{id}/preview", dataSetId).then().statusCode(OK.value());
}

Example 30 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class DataSetServiceTest method compatibleDatasetsListDateSort.

@Test
public void compatibleDatasetsListDateSort() throws Exception {
    String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-13");
    String dataSetId2 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-12");
    String dataSetId3 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-11");
    DataSetMetadata metadata1 = dataSetMetadataRepository.get(dataSetId);
    metadata1.setName("CCCC");
    dataSetMetadataRepository.save(metadata1);
    DataSetMetadata metadata2 = dataSetMetadataRepository.get(dataSetId2);
    metadata2.setName("BBBB");
    dataSetMetadataRepository.save(metadata2);
    DataSetMetadata metadata3 = dataSetMetadataRepository.get(dataSetId3);
    metadata3.setName("AAAA");
    dataSetMetadataRepository.save(metadata3);
    // when
    final String actual = expect().statusCode(200).log().ifValidationFails().get("/datasets/{id}/compatibledatasets?sort=creationDate", dataSetId).asString();
    // Ensure order by name (most recent first)
    final Iterator<JsonNode> elements = mapper.readTree(actual).elements();
    String[] expectedNames = new String[] { "AAAA", "BBBB" };
    int i = 0;
    while (elements.hasNext()) {
        assertThat(elements.next().get("name").asText(), is(expectedNames[i++]));
    }
}