Examples with Statistics org.talend.dataprep.api.dataset.statistics.Statistics used on opensource projects

Example 1 with Statistics

use of org.talend.dataprep.api.dataset.statistics.Statistics in project data-prep by Talend.

the class ReorderColumn method swapColumnMetadata.

protected void swapColumnMetadata(ColumnMetadata originColumn, ColumnMetadata targetColumn) throws Exception {
    ColumnMetadata targetColumnCopy = ColumnMetadata.Builder.column().copy(targetColumn).build();
    ColumnMetadata originColumnCopy = ColumnMetadata.Builder.column().copy(originColumn).build();
    BeanUtils.copyProperties(targetColumn, originColumn);
    BeanUtils.copyProperties(originColumn, targetColumnCopy);
    Statistics originalStatistics = originColumnCopy.getStatistics();
    Statistics targetStatistics = targetColumnCopy.getStatistics();
    BeanUtils.copyProperties(targetColumn.getStatistics(), originalStatistics);
    BeanUtils.copyProperties(originColumn.getStatistics(), targetStatistics);
    Quality originalQuality = originColumnCopy.getQuality();
    Quality targetQualityCopty = targetColumnCopy.getQuality();
    BeanUtils.copyProperties(targetColumn.getQuality(), originalQuality);
    BeanUtils.copyProperties(originColumn.getQuality(), targetQualityCopty);
}

Example 2 with Statistics

use of org.talend.dataprep.api.dataset.statistics.Statistics in project data-prep by Talend.

the class ExtractDateTokensTest method should_update_metadata.

@Test
public void should_update_metadata() throws IOException {
    // given
    final List<ColumnMetadata> input = new ArrayList<>();
    input.add(createMetadata("0000"));
    input.add(createMetadata("0001"));
    input.add(createMetadata("0002"));
    final RowMetadata rowMetadata = new RowMetadata(input);
    ObjectMapper mapper = new ObjectMapper();
    final Statistics statistics = mapper.reader(Statistics.class).readValue(getDateTestJsonAsStream("statistics_yyyy-MM-dd.json"));
    input.get(1).setStatistics(statistics);
    // when
    ActionTestWorkbench.test(rowMetadata, actionRegistry, factory.create(action, parameters));
    // then
    assertNotNull(rowMetadata.getById("0003"));
    assertNotNull(rowMetadata.getById("0004"));
    assertNotNull(rowMetadata.getById("0005"));
    assertNotNull(rowMetadata.getById("0006"));
    assertNull(rowMetadata.getById("0007"));
}

Example 3 with Statistics

use of org.talend.dataprep.api.dataset.statistics.Statistics in project data-prep by Talend.

the class SplitTest method test_TDP_876.

@Test
public void test_TDP_876() {
    // given
    final DataSetRow row = // 
    builder().with(// 
    value("lorem bacon").type(Type.STRING)).with(// 
    value("Bacon ipsum dolor amet swine leberkas pork belly").type(Type.STRING)).with(// 
    value("01/01/2015").type(Type.STRING)).build();
    // when
    // 
    ActionTestWorkbench.test(// 
    Collections.singletonList(row), // Test requires some analysis in asserts
    analyzerService, actionRegistry, factory.create(action, parameters));
    // then
    final RowMetadata actual = row.getRowMetadata();
    Statistics originalStats = actual.getById("0001").getStatistics();
    final List<PatternFrequency> originalPatterns = originalStats.getPatternFrequencies();
    assertFalse(originalPatterns.equals(actual.getById("0003").getStatistics().getPatternFrequencies()));
    assertFalse(originalPatterns.equals(actual.getById("0004").getStatistics().getPatternFrequencies()));
}

Example 4 with Statistics

use of org.talend.dataprep.api.dataset.statistics.Statistics in project data-prep by Talend.

the class ColumnMetadataTest method should_set_empty_statistics.

@Test
public void should_set_empty_statistics() {
    ColumnMetadata column = ColumnMetadata.Builder.column().id(42).type(Type.STRING).build();
    column.setStatistics(null);
    assertEquals(new Statistics(), column.getStatistics());
}

Example 5 with Statistics

use of org.talend.dataprep.api.dataset.statistics.Statistics in project data-prep by Talend.

the class DataSetServiceTest method datePattern.

@Test
public void datePattern() throws Exception {
    int before = dataSetMetadataRepository.size();
    String dataSetId = given().body(IOUtils.toString(this.getClass().getResourceAsStream("../date_time_pattern.csv"), UTF_8)).queryParam(CONTENT_TYPE, "text/csv").when().post("/datasets").asString();
    int after = dataSetMetadataRepository.size();
    assertThat(after - before, is(1));
    assertQueueMessages(dataSetId);
    final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId);
    assertNotNull(dataSetMetadata);
    final ColumnMetadata column = dataSetMetadata.getRowMetadata().getById("0001");
    assertThat(column.getType(), is("date"));
    assertThat(column.getDomain(), is(""));
    final Statistics statistics = mapper.readerFor(Statistics.class).readValue(this.getClass().getResourceAsStream("../date_time_pattern_expected.json"));
    assertThat(column.getStatistics(), CoreMatchers.equalTo(statistics));
}