Examples with DataSetMetadata org.talend.dataprep.api.dataset.DataSetMetadata used on opensource projects

Example 6 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class DataSetAPITest method testDataSetUpdateMetadata.

@Test
public void testDataSetUpdateMetadata() throws Exception {
    // given
    final String dataSetId = testClient.createDataset("dataset/dataset.csv", "tagada");
    // when
    final String jsonOriginalMetadata = when().get("/api/datasets/{id}/metadata", dataSetId).asString();
    final DataSetMetadata metadata = mapper.readValue(jsonOriginalMetadata, DataSetMetadata.class);
    metadata.setName("Toto");
    final String jsonMetadata = mapper.writeValueAsString(metadata);
    given().body(jsonMetadata).when().put("/api/datasets/{id}/metadata", dataSetId).asString();
    final String jsonUpdatedMetadata = when().get("/api/datasets/{id}/metadata", dataSetId).asString();
    final DataSetMetadata updatedMetadata = mapper.readValue(jsonUpdatedMetadata, DataSetMetadata.class);
    // then
    assertEquals(updatedMetadata, metadata);
}

Example 7 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class DataSetAPITest method should_list_filtered_datasets_properly.

@Test
public void should_list_filtered_datasets_properly() throws Exception {
    // create data sets
    final String dataSetId1 = testClient.createDataset("dataset/dataset.csv", "dataset1");
    final String dataSetId2 = testClient.createDataset("dataset/dataset.csv", "dataset2");
    final String dataSetId3 = testClient.createDataset("dataset/dataset.csv", "dataset3");
    testClient.createDataset("dataset/dataset.csv", "dataset4");
    // Make dataset1 more recent
    final DataSetMetadata dataSetMetadata1 = dataSetMetadataRepository.get(dataSetId1);
    dataSetMetadata1.getGovernance().setCertificationStep(DataSetGovernance.Certification.CERTIFIED);
    dataSetMetadata1.setLastModificationDate((now().getEpochSecond() + 1) * 1_000);
    dataSetMetadataRepository.save(dataSetMetadata1);
    final DataSetMetadata dataSetMetadata2 = dataSetMetadataRepository.get(dataSetId2);
    dataSetMetadataRepository.save(dataSetMetadata2);
    final DataSetMetadata dataSetMetadata3 = dataSetMetadataRepository.get(dataSetId3);
    dataSetMetadata3.getGovernance().setCertificationStep(DataSetGovernance.Certification.CERTIFIED);
    dataSetMetadataRepository.save(dataSetMetadata3);
    // add favorite
    UserData userData = new UserData(security.getUserId(), versionService.version().getVersionId());
    HashSet<String> favorites = new HashSet<>();
    favorites.add(dataSetMetadata1.getId());
    favorites.add(dataSetMetadata2.getId());
    userData.setFavoritesDatasets(favorites);
    userDataRepository.save(userData);
    // @formatter:off
    // certified, favorite and recent
    given().queryParam("favorite", "true").queryParam("certified", "true").queryParam("limit", "true").queryParam("name", "dataset").when().get("/api/datasets").then().statusCode(200).body("name", hasItem("dataset1")).body("name", hasSize(1));
    // certified, favorite and recent
    given().queryParam("favorite", "true").queryParam("certified", "true").queryParam("limit", "true").queryParam("name", "2").when().get("/api/datasets").then().statusCode(200).body("name", hasSize(0));
    // only names
    given().queryParam("name", "ATASET2").when().get("/api/datasets").then().statusCode(200).body("name", hasItem("dataset2")).body("name", hasSize(1));
    // only favorites
    given().queryParam("favorite", "true").when().get("/api/datasets").then().statusCode(200).body("name", hasItems("dataset1", "dataset2")).body("name", hasSize(2));
    // only certified
    given().queryParam("certified", "true").when().get("/api/datasets").then().statusCode(200).body("name", hasItems("dataset1", "dataset3")).body("name", hasSize(2));
    // only recent
    given().queryParam("limit", "true").when().get("/api/datasets").then().statusCode(200).body("name", hasItems("dataset2", "dataset3", "dataset4")).body("name", hasSize(3));
    // all
    when().get("/api/datasets").then().statusCode(200).body("name", hasItems("dataset1", "dataset2", "dataset3", "dataset4")).body("name", hasSize(4));
// @formatter:on
}

Example 8 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class PreparationAPITest method testPreparationInitialMetadata.

@Test
public void testPreparationInitialMetadata() throws Exception {
    // given
    final String preparationName = "testPreparationContentGet";
    final String preparationId = testClient.createPreparationFromFile("dataset/dataset.csv", preparationName, home.getId());
    // when
    final DataSetMetadata actual = testClient.getPrepMetadata(preparationId);
    // then
    assertNotNull(actual);
    final List<ColumnMetadata> columns = actual.getRowMetadata().getColumns();
    assertEquals(6, columns.size());
    final List<String> expectedColumns = Arrays.asList("id", "firstname", "lastname", "age", "date-of-birth", "alive");
    for (ColumnMetadata column : columns) {
        assertTrue(expectedColumns.contains(column.getName()));
    }
}

Example 9 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class APIPreparationConversionsTest method shouldEnrichPreparationWithDataset.

@Test
public void shouldEnrichPreparationWithDataset() {
    // given
    DataSetMetadata metadata = getDataSetMetadata("super dataset", 1001L);
    setupHystrixCommand(DataSetGetMetadata.class, metadata);
    final PreparationMessage preparation = getPreparationMessage(metadata.getId());
    Folder folder = getFolder("F-753854");
    setupHystrixCommand(LocatePreparation.class, folder);
    // when
    final EnrichedPreparation actual = conversionService.convert(preparation, EnrichedPreparation.class);
    // then
    assertEquals(metadata.getId(), actual.getSummary().getDataSetId());
    assertEquals(metadata.getName(), actual.getSummary().getDataSetName());
    assertEquals(metadata.getContent().getNbRecords(), actual.getSummary().getDataSetNbRow());
    final List<String> expectedSteps = preparation.getSteps().stream().map(Step::getId).collect(Collectors.toList());
    final List<String> actualSteps = actual.getSteps();
    assertNotNull(actualSteps);
    assertEquals(expectedSteps.size(), expectedSteps.size());
    expectedSteps.forEach(s -> assertTrue(actualSteps.contains(s)));
    assertEquals(folder, actual.getFolder());
}

Example 10 with DataSetMetadata

use of org.talend.dataprep.api.dataset.DataSetMetadata in project data-prep by Talend.

the class PipelineTest method testPipeline.

@Test
public void testPipeline() throws Exception {
    // given
    final Pipeline pipeline = new Pipeline(NodeBuilder.source().to(output).build());
    final RowMetadata rowMetadata = new RowMetadata();
    final DataSetRow row1 = new DataSetRow(rowMetadata);
    final DataSetRow row2 = new DataSetRow(rowMetadata);
    final List<DataSetRow> records = new ArrayList<>();
    records.add(row1);
    records.add(row2);
    final DataSet dataSet = new DataSet();
    final DataSetMetadata metadata = new DataSetMetadata();
    metadata.setRowMetadata(rowMetadata);
    dataSet.setMetadata(metadata);
    dataSet.setRecords(records.stream());
    // when
    pipeline.execute(dataSet);
    // then
    assertThat(output.getCount(), is(2));
    assertThat(output.getRow(), is(row2));
    assertThat(output.getMetadata(), is(rowMetadata));
    assertThat(output.getSignal(), is(END_OF_STREAM));
}