Examples with DataSetMetadata org.talend.dataprep.api.dataset.DataSetMetadata used on opensource projects

Example 71 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class DataSetServiceTest method updateRawContent_should_preserve_non_content_related_metadata_except_last_modification_date.

@Test
public void updateRawContent_should_preserve_non_content_related_metadata_except_last_modification_date() throws Exception {
    // given
    String dataSetId = UUID.randomUUID().toString();
    DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id(dataSetId).build();
    dataSetMetadataRepository.save(dataSetMetadata);
    given().body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)).when().put("/datasets/{id}/raw", dataSetId).then().statusCode(OK.value());
    String datasets = when().get("/datasets").asString();
    List<DataSetMetadata> datasetsMetadata = mapper.readValue(datasets, new TypeReference<ArrayList<DataSetMetadata>>() {
    });
    final DataSetMetadata original = datasetsMetadata.get(0);
    // when
    given().body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA2_CSV), UTF_8)).when().put("/datasets/{id}/raw", dataSetId).then().statusCode(OK.value());
    // then
    datasets = when().get("/datasets").asString();
    datasetsMetadata = mapper.readValue(datasets, new TypeReference<ArrayList<DataSetMetadata>>() {
    });
    final DataSetMetadata copy = datasetsMetadata.get(0);
    assertThat(copy.getId(), equalTo(original.getId()));
    assertThat(copy.getAppVersion(), equalTo(original.getAppVersion()));
    assertThat(copy.getAuthor(), equalTo(original.getAuthor()));
    assertThat(copy.getCreationDate(), equalTo(original.getCreationDate()));
    assertThat(copy.getLocation(), equalTo(original.getLocation()));
}

Example 72 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class XlsSerializerTest method getDataSetMetadataFromExcelFile.

/**
 * Return the dataset metadata out of the given file name.
 *
 * @param fileName the excel file to open.
 * @param sheetName name of the excel sheet.
 * @return the dataset metadata.
 * @throws IOException s**y happens.
 */
private DataSetMetadata getDataSetMetadataFromExcelFile(String fileName, String sheetName) throws IOException {
    DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id("ff").sheetName(sheetName).build();
    try (InputStream inputStream = this.getClass().getResourceAsStream(fileName)) {
        List<Schema.SheetContent> sheetContents = xlsSchemaParser.parseAllSheets(getRequest(inputStream, "#8"));
        List<ColumnMetadata> columnsMetadata = sheetContents.get(0).getColumnMetadatas();
        logger.debug("columnsMetadata: {}", columnsMetadata);
        Assertions.assertThat(columnsMetadata).isNotNull().isNotEmpty();
        dataSetMetadata.getRowMetadata().setColumns(columnsMetadata);
    }
    return dataSetMetadata;
}

Example 73 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class XlsSerializerTest method test_second_sheet_parsing.

@Test
public void test_second_sheet_parsing() throws Exception {
    String fileName = "Talend_Desk-Tableau-Bord-011214.xls";
    assertFormat(fileName);
    DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id("beer").sheetName("Leads").build();
    try (InputStream inputStream = this.getClass().getResourceAsStream(fileName)) {
        List<Schema.SheetContent> sheetContents = xlsSchemaParser.parseAllSheets(getRequest(inputStream, "#8"));
        List<ColumnMetadata> columnsMetadata = sheetContents.stream().filter(sheetContent -> "Leads".equals(sheetContent.getName())).findFirst().get().getColumnMetadatas();
        logger.debug("columnsMetadata: {}", columnsMetadata);
        Assertions.assertThat(columnsMetadata).isNotNull().isNotEmpty().hasSize(6);
        ColumnMetadata columnMetadata = columnsMetadata.get(4);
        Assertions.assertThat(columnMetadata.getHeaderSize()).isEqualTo(1);
        Assertions.assertThat(columnMetadata.getName()).isEqualTo("age");
        Assertions.assertThat(columnMetadata.getType()).isEqualTo(Type.STRING.getName());
        dataSetMetadata.getRowMetadata().setColumns(columnsMetadata);
    }
    List<Map<String, String>> values = getValuesFromFile(fileName, dataSetMetadata);
    logger.trace("values: {}", values);
    Assertions.assertThat(values).isNotEmpty().hasSize(3);
    // 
    Assertions.assertThat(values.get(0)).contains(// 
    entry("0000", "301638"), // 
    entry("0001", "foo@foo.com"), entry("0004", "23"));
    // 
    Assertions.assertThat(values.get(1)).contains(// 
    entry("0000", "12349383"), // 
    entry("0001", "beer@gof.org"), entry("0004", "23"));
    // 
    Assertions.assertThat(values.get(2)).contains(// 
    entry("0000", "73801093"), // 
    entry("0001", "wine@go.com"), entry("0003", "Jean"));
}

Example 74 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class XlsSerializerTest method more_than_52_columns.

/**
 * test for TDP-1660 and TDP-1648
 */
@Test
public void more_than_52_columns() throws Exception {
    String fileName = "TDP_Epics.xlsx";
    DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id("epics").build();
    try (InputStream inputStream = this.getClass().getResourceAsStream(fileName)) {
        // 
        dataSetMetadata.getRowMetadata().setColumns(// 
        xlsSchemaParser.parse(getRequest(inputStream, "#456")).getSheetContents().get(// 
        0).getColumnMetadatas());
    }
    // 
    Assertions.assertThat(dataSetMetadata.getRowMetadata().getColumns()).isNotNull().isNotEmpty().hasSize(98);
    List<Map<String, String>> values = getValuesFromFile(fileName, dataSetMetadata);
    logger.debug("values: {}", values);
    Assertions.assertThat(values.get(3).get("0090")).isEqualTo("Small");
}

Example 75 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class SortAndOrderHelperTest method getDatasetComparison.

private // 
int getDatasetComparison(// 
String firstName, // 
String secondName, // 
String firstOwnerFirstName, // 
String secondOwnerFirstName, // 
String firstOwnerLastName, // 
String secondOwnerLastName, // 
long firstCreation, // 
long secondCreation, // 
long firstModification, // 
long secondModification, // 
long firstSize, // 
long secondSize, Sort sort, Order order) {
    DataSetMetadata firstUserDatasetMetadata = createUserDatasetMetadata("firstDsId", firstName, "1", firstCreation, firstModification, null, null, firstOwnerFirstName, firstOwnerLastName);
    firstUserDatasetMetadata.getContent().setNbRecords(firstSize);
    DataSetMetadata secondUserDatasetMetadata = createUserDatasetMetadata("secondDsId", secondName, "2", secondCreation, secondModification, null, null, secondOwnerFirstName, secondOwnerLastName);
    secondUserDatasetMetadata.getContent().setNbRecords(secondSize);
    // Used to make sure that when using DatasetMetadata instead of UserDatasetMetada the behaviour of the comparison remains
    // the same
    DataSetMetadata firstDatasetMetadata = createDatasetMetadata("firstDsId", firstName, firstCreation, firstModification, null, null, firstOwnerFirstName, firstOwnerLastName);
    firstDatasetMetadata.getContent().setNbRecords(firstSize);
    DataSetMetadata secondDatasetMetadata = createDatasetMetadata("secondDsId", secondName, secondCreation, secondModification, null, null, secondOwnerFirstName, secondOwnerLastName);
    secondDatasetMetadata.getContent().setNbRecords(secondSize);
    // when
    Comparator<DataSetMetadata> preparationComparator = SortAndOrderHelper.getDataSetMetadataComparator(sort, order);
    // then
    assertNotNull(preparationComparator);
    final int userDatasetMetadataOrder = preparationComparator.compare(firstUserDatasetMetadata, secondUserDatasetMetadata);
    final int datasetMetadataOrder = preparationComparator.compare(firstDatasetMetadata, secondDatasetMetadata);
    // Make sure that when using DatasetMetadata instead of UserDatasetMetada the behaviour of the comparison remains the same
    assertEquals(userDatasetMetadataOrder, datasetMetadataOrder);
    return userDatasetMetadataOrder;
}