Examples with DatasetContent org.talend.dataprep.qa.dto.DatasetContent used on opensource projects

Example 1 with DatasetContent

use of org.talend.dataprep.qa.dto.DatasetContent in project data-prep by Talend.

the class PreparationStep method thePreparationShouldHaveThefollowingQualityBar.

@Then("^The preparation \"(.*)\" should have the following quality bar characteristics on the column number \"(.*)\":$")
public void thePreparationShouldHaveThefollowingQualityBar(String preparationName, String columnNumber, DataTable dataTable) throws Exception {
    Response response = api.getPreparationContent(context.getPreparationId(suffixName(preparationName)), VERSION_HEAD, HEAD_ID, StringUtils.EMPTY);
    response.then().statusCode(OK.value());
    DatasetContent datasetContent = response.as(DatasetContent.class);
    final Map<String, String> parameters = dataTable.asMap(String.class, String.class);
    Integer validExpected = Integer.parseInt(parameters.get(VALID_CELL));
    Integer invalidExpected = Integer.parseInt(parameters.get(INVALID_CELL));
    Integer emptyExpected = Integer.parseInt(parameters.get(EMPTY_CELL));
    ContentMetadataColumn columnMetadata = datasetContent.metadata.columns.get(Integer.parseInt(columnNumber));
    assertEquals(validExpected, columnMetadata.quality.get(VALID_CELL));
    assertEquals(invalidExpected, columnMetadata.quality.get(INVALID_CELL));
    assertEquals(emptyExpected, columnMetadata.quality.get(EMPTY_CELL));
}

Example 2 with DatasetContent

use of org.talend.dataprep.qa.dto.DatasetContent in project data-prep by Talend.

the class PreparationStep method thePreparationShouldHaveThefollowingTypeOnThefollowingColumn.

@Then("^The preparation \"(.*)\" should have the following type \"(.*)\" on the following column \"(.*)\"$")
public void thePreparationShouldHaveThefollowingTypeOnThefollowingColumn(String preparationName, String columnType, String columnNumber) throws Exception {
    Response response = api.getPreparationContent(context.getPreparationId(suffixName(preparationName)), VERSION_HEAD, HEAD_ID, StringUtils.EMPTY);
    response.then().statusCode(OK.value());
    DatasetContent datasetContent = response.as(DatasetContent.class);
    ContentMetadataColumn columnMetadata = datasetContent.metadata.columns.get(Integer.parseInt(columnNumber));
    assertEquals(columnType, columnMetadata.type);
}

Example 3 with DatasetContent

use of org.talend.dataprep.qa.dto.DatasetContent in project data-prep by Talend.

the class PreparationStep method thePreparationShouldHaveThefollowingInvalidCells.

@Then("^The preparation \"(.*)\" should have the following invalid characteristics on the row number \"(.*)\":$")
public void thePreparationShouldHaveThefollowingInvalidCells(String preparationName, String columnNumber, DataTable dataTable) throws Exception {
    Response response = api.getPreparationContent(context.getPreparationId(suffixName(preparationName)), VERSION_HEAD, HEAD_ID, StringUtils.EMPTY);
    response.then().statusCode(OK.value());
    DatasetContent datasetContent = response.as(DatasetContent.class);
    final Map<String, String> parameters = dataTable.asMap(String.class, String.class);
    String invalidCells = parameters.get("invalidCells");
    HashMap values = (HashMap<String, String>) datasetContent.records.get(Integer.parseInt(columnNumber));
    if (!invalidCells.equals(StringUtils.EMPTY)) {
        assertEquals(invalidCells, values.get(TDP_INVALID_MARKER));
    } else {
        // there is no invalid cell
        assertNull(values.get(TDP_INVALID_MARKER));
    }
}

Example 4 with DatasetContent

use of org.talend.dataprep.qa.dto.DatasetContent in project data-prep by Talend.

the class DataPrepStep method getDatasetContent.

/**
 * Returns the dataset content, once all DQ analysis are done and so all fields are up-to-date.
 *
 * @param datasetId the id of the dataset
 * @param tql the TQL filter to apply to the dataset
 * @return the up-to-date dataset content
 */
protected DatasetContent getDatasetContent(String datasetId, String tql) throws Exception {
    AtomicReference<DatasetContent> datasetContentReference = new AtomicReference<>();
    // TODO I guess this wait is useless since we use {DataPrepStep#checkDatasetMetadataStatus} before
    api.waitResponse("Waiting frequency table from dataset metadata of " + datasetId).until(() -> {
        Response response = api.getDataset(datasetId, tql);
        response.then().statusCode(200);
        DatasetContent datasetContent = response.as(DatasetContent.class);
        datasetContentReference.set(datasetContent);
        return // 
        datasetContent.metadata.columns.stream().findFirst().orElse(new ContentMetadataColumn()).statistics.frequencyTable;
    }, is(not(empty())));
    return datasetContentReference.get();
}

Example 5 with DatasetContent

use of org.talend.dataprep.qa.dto.DatasetContent in project data-prep by Talend.

the class FilterStep method checkFilterAppliedOnDataSet.

@Then("^The characteristics of the dataset \"(.*)\" match:$")
public void checkFilterAppliedOnDataSet(String datasetName, DataTable dataTable) throws Exception {
    Map<String, String> expected = dataTable.asMap(String.class, String.class);
    DatasetContent datasetContent = (DatasetContent) context.getObject("dataSetContent");
    if (datasetContent == null) {
        datasetContent = getDatasetContent(context.getDatasetId(suffixName(datasetName)), null);
    }
    if (expected.get("records") != null) {
        checkRecords(datasetContent.records, expected.get("records"));
    }
    if (expected.get("sample_records_count") != null) {
        checkSampleRecordsCount(datasetContent.metadata.records, expected.get("sample_records_count"));
    }
    if (expected.get("quality") != null) {
        checkQualityPerColumn(datasetContent.metadata.columns, expected.get("quality"));
    }
}