Examples with DataSetMetadata org.talend.dataprep.api.dataset.DataSetMetadata used on opensource projects

Example 36 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class StatisticsAnalysisTest method should_update_dataset_nb_records.

@Test
public void should_update_dataset_nb_records() throws Exception {
    // given
    final InputStream dataset = this.getClass().getResourceAsStream("dataset.csv");
    // when
    final DataSetMetadata metadata = initializeDataSetMetadata(dataset);
    // then
    assertThat(metadata.getContent().getNbRecords(), is(4L));
}

Example 37 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class ContentAnalysisTest method testAnalysisWithLimit.

@Test
public void testAnalysisWithLimit() {
    final DataSetMetadata metadata = createMetadata(UUID.randomUUID().toString(), Arrays.asList("id", "first_name", "last_name", "email", "gender", "ip_address"));
    metadataBuilder.metadata().id("3548").build();
    createCsvDataSet(metadata, "100_lines.csv");
    final Long newLimit = 16L;
    final Long originalLimit = (Long) ReflectionTestUtils.getField(contentAnalysis, ContentAnalysis.class, "sizeLimit");
    ReflectionTestUtils.setField(contentAnalysis, "sizeLimit", newLimit);
    contentAnalysis.analyze(metadata.getId());
    ReflectionTestUtils.setField(contentAnalysis, "sizeLimit", originalLimit);
    final DataSetMetadata actual = dataSetMetadataRepository.get(metadata.getId());
    final Optional<Long> limit = actual.getContent().getLimit();
    assertTrue(limit.isPresent());
    assertThat(limit.get(), is(newLimit));
}

Example 38 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class ContentAnalysisTest method testAnalysisWithoutLimit.

@Test
public void testAnalysisWithoutLimit() {
    final DataSetMetadata metadata = metadataBuilder.metadata().id(UUID.randomUUID().toString()).build();
    createCsvDataSet(metadata, "5_lines.csv");
    contentAnalysis.analyze(metadata.getId());
    final DataSetMetadata actual = dataSetMetadataRepository.get(metadata.getId());
    assertFalse(actual.getContent().getLimit().isPresent());
}

Example 39 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class FormatAnalysisTest method testCSVAnalysis.

@Test
public void testCSVAnalysis() {
    String id = UUID.randomUUID().toString();
    final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build();
    dataSetMetadataRepository.save(metadata);
    contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../avengers.csv"));
    formatAnalysis.analyze(id);
    final DataSetMetadata actual = dataSetMetadataRepository.get(id);
    assertThat(actual, notNullValue());
    assertThat(actual.getContent().getFormatFamilyId(), is(CSVFormatFamily.BEAN_ID));
    assertThat(actual.getContent().getMediaType(), is("text/csv"));
    assertThat(actual.getContent().getParameters().get("SEPARATOR"), is(";"));
}

Example 40 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class FormatAnalysisTest method testEncodingDetection_UTF16LE_WithoutBOM.

/**
 * see https://jira.talendforge.org/browse/TDP-2930.
 */
@Test
public void testEncodingDetection_UTF16LE_WithoutBOM() {
    String id = UUID.randomUUID().toString();
    final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build();
    dataSetMetadataRepository.save(metadata);
    contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../utf16_LE_without_bom.txt"));
    formatAnalysis.analyze(id);
    final DataSetMetadata actual = dataSetMetadataRepository.get(id);
    assertThat(actual, notNullValue());
    assertThat(actual.getContent().getFormatFamilyId(), is(CSVFormatFamily.BEAN_ID));
    assertThat(actual.getContent().getMediaType(), is("text/csv"));
    assertThat(actual.getContent().getParameters().get("SEPARATOR"), is(","));
    assertThat(actual.getEncoding(), is("UTF-16LE"));
}