Examples with RowMetadata org.talend.dataprep.api.dataset.RowMetadata used on opensource projects

Example 86 with RowMetadata

use of org.talend.dataprep.api.dataset.RowMetadata in project data-prep by Talend.

the class DataSetService method preview.

/**
 * Returns preview of the the data set content for given id (first 100 rows). Service might return
 * {@link org.apache.http.HttpStatus#SC_ACCEPTED} if the data set exists but analysis is not yet fully
 * completed so content is not yet ready to be served.
 *
 * @param metadata If <code>true</code>, includes data set metadata information.
 * @param sheetName the sheet name to preview
 * @param dataSetId A data set id.
 */
@RequestMapping(value = "/datasets/{id}/preview", method = RequestMethod.GET)
@ApiOperation(value = "Get a data preview set by id", notes = "Get a data set preview content based on provided id. Not valid or non existing data set id returns empty content. Data set not in drat status will return a redirect 301")
@Timed
@ResponseBody
public DataSet preview(@RequestParam(defaultValue = "true") @ApiParam(name = "metadata", value = "Include metadata information in the response") boolean metadata, @RequestParam(defaultValue = "") @ApiParam(name = "sheetName", value = "Sheet name to preview") String sheetName, @PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the requested data set") String dataSetId) {
    DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId);
    if (dataSetMetadata == null) {
        HttpResponseContext.status(HttpStatus.NO_CONTENT);
        // No data set, returns empty content.
        return DataSet.empty();
    }
    if (!dataSetMetadata.isDraft()) {
        // Moved to get data set content operation
        HttpResponseContext.status(HttpStatus.MOVED_PERMANENTLY);
        HttpResponseContext.header("Location", "/datasets/" + dataSetId + "/content");
        // dataset not anymore a draft so preview doesn't make sense.
        return DataSet.empty();
    }
    if (StringUtils.isNotEmpty(sheetName)) {
        dataSetMetadata.setSheetName(sheetName);
    }
    // take care of previous data without schema parser result
    if (dataSetMetadata.getSchemaParserResult() != null) {
        // sheet not yet set correctly so use the first one
        if (StringUtils.isEmpty(dataSetMetadata.getSheetName())) {
            String theSheetName = dataSetMetadata.getSchemaParserResult().getSheetContents().get(0).getName();
            LOG.debug("preview for dataSetMetadata: {} with sheetName: {}", dataSetId, theSheetName);
            dataSetMetadata.setSheetName(theSheetName);
        }
        String theSheetName = dataSetMetadata.getSheetName();
        Optional<Schema.SheetContent> sheetContentFound = dataSetMetadata.getSchemaParserResult().getSheetContents().stream().filter(sheetContent -> theSheetName.equals(sheetContent.getName())).findFirst();
        if (!sheetContentFound.isPresent()) {
            HttpResponseContext.status(HttpStatus.NO_CONTENT);
            // No sheet found, returns empty content.
            return DataSet.empty();
        }
        List<ColumnMetadata> columnMetadatas = sheetContentFound.get().getColumnMetadatas();
        if (dataSetMetadata.getRowMetadata() == null) {
            dataSetMetadata.setRowMetadata(new RowMetadata(emptyList()));
        }
        dataSetMetadata.getRowMetadata().setColumns(columnMetadatas);
    } else {
        LOG.warn("dataset#{} has draft status but any SchemaParserResult");
    }
    // Build the result
    DataSet dataSet = new DataSet();
    if (metadata) {
        dataSet.setMetadata(conversionService.convert(dataSetMetadata, UserDataSetMetadata.class));
    }
    dataSet.setRecords(contentStore.stream(dataSetMetadata).limit(100));
    return dataSet;
}

Example 87 with RowMetadata

use of org.talend.dataprep.api.dataset.RowMetadata in project data-prep by Talend.

the class ToPEPersistentIdentifiable method run.

@Override
public void run() {
    LOGGER.debug("starting upgrade from {} to {}.", Step.class, PersistentStep.class);
    final AtomicLong counter = new AtomicLong(0L);
    fileSystemPreparationRepository.list(Step.class).forEach(s -> {
        fileSystemPreparationRepository.remove(s);
        PersistentStep persistentStep = turnToPersistentStep(s);
        preparationRepository.add(persistentStep);
        LOGGER.debug("step {} updated to {}", s, persistentStep);
        counter.incrementAndGet();
    });
    LOGGER.info("Upgrade from {} to {} done, {} steps processed.", Step.class, PersistentStep.class, counter.get());
    LOGGER.debug("starting upgrade from {} to {}.", Preparation.class, PersistentPreparation.class);
    final Stream<Preparation> preparations = fileSystemPreparationRepository.list(Preparation.class);
    preparations.forEach(p -> {
        fileSystemPreparationRepository.remove(p);
        PersistentPreparation persistentPreparation = turnToPersistentPreparation(p);
        preparationRepository.add(persistentPreparation);
    });
    LOGGER.info("Upgrade from {} to {} done.", Preparation.class, PersistentPreparation.class);
    LOGGER.info("Migration of step ids in preparation...");
    final Stream<PersistentPreparation> persistentPreparations = preparationRepository.list(PersistentPreparation.class);
    persistentPreparations.forEach(p -> {
        LOGGER.info("Migration of preparation #{}", p.getId());
        final List<String> stepsIds = preparationUtils.listStepsIds(p.getHeadId(), preparationRepository);
        p.setSteps(stepsIds);
        final DataSetMetadata metadata = dataSetMetadataRepository.get(p.getDataSetId());
        if (metadata != null) {
            LOGGER.info("Set metadata {} in preparation {}.", p.getDataSetId(), p.getId());
            p.setRowMetadata(metadata.getRowMetadata());
        } else {
            LOGGER.info("Metadata {} not found for preparation {}.", p.getDataSetId(), p.getId());
            p.setRowMetadata(new RowMetadata());
        }
        preparationRepository.add(p);
        LOGGER.info("Migration of preparation #{} done ({} steps)", p.getId(), stepsIds.size());
    });
    LOGGER.info("Migration of step ids in preparation done.");
}

Example 88 with RowMetadata

use of org.talend.dataprep.api.dataset.RowMetadata in project data-prep by Talend.

the class FillWithIntegerIfEmptyTest method test_apply_inplace.

@Test
public void test_apply_inplace() throws Exception {
    // given
    final Map<String, String> values = new HashMap<>();
    values.put("0000", "David Bowie");
    values.put("0001", "");
    values.put("0002", "Something");
    final DataSetRow row = new DataSetRow(values);
    final RowMetadata rowMetadata = row.getRowMetadata();
    rowMetadata.getById("0001").setType(Type.INTEGER.getName());
    Map<String, String> parameters = // 
    ActionMetadataTestUtils.parseParameters(this.getClass().getResourceAsStream("fillEmptyIntegerAction.json"));
    // when
    ActionTestWorkbench.test(row, actionRegistry, factory.create(action, parameters));
    // then
    Assert.assertEquals("25", row.get("0001"));
    Assert.assertEquals("David Bowie", row.get("0000"));
}

Example 89 with RowMetadata

use of org.talend.dataprep.api.dataset.RowMetadata in project data-prep by Talend.

the class FillWithIntegerIfEmptyTest method should_fill_empty_integer_other_column.

@Test
public void should_fill_empty_integer_other_column() throws Exception {
    // given
    final Map<String, String> values = new HashMap<>();
    values.put("0000", "David Bowie");
    values.put("0001", "");
    values.put("0002", "10");
    final DataSetRow row = new DataSetRow(values);
    final RowMetadata rowMetadata = row.getRowMetadata();
    rowMetadata.getById("0001").setType(Type.INTEGER.getName());
    rowMetadata.getById("0002").setType(Type.INTEGER.getName());
    Map<String, String> parameters = // 
    ActionMetadataTestUtils.parseParameters(this.getClass().getResourceAsStream("fillEmptyIntegerAction.json"));
    // when
    parameters.put(FillIfEmpty.MODE_PARAMETER, FillIfEmpty.OTHER_COLUMN_MODE);
    parameters.put(FillIfEmpty.SELECTED_COLUMN_PARAMETER, "0002");
    ActionTestWorkbench.test(row, actionRegistry, factory.create(action, parameters));
    // then
    Assert.assertEquals("10", row.get("0001"));
    Assert.assertEquals("David Bowie", row.get("0000"));
}

Example 90 with RowMetadata

use of org.talend.dataprep.api.dataset.RowMetadata in project data-prep by Talend.

the class FillWithIntegerTest method should_not_fill_empty_integer.

@Test
public void should_not_fill_empty_integer() throws Exception {
    // given
    final Map<String, String> values = new HashMap<>();
    values.put("0000", "David Bowie");
    values.put("0001", "not empty");
    values.put("0002", "Something");
    final DataSetRow row = new DataSetRow(values);
    final RowMetadata rowMetadata = row.getRowMetadata();
    rowMetadata.getById("0001").setType(Type.INTEGER.getName());
    Map<String, String> parameters = // 
    ActionMetadataTestUtils.parseParameters(this.getClass().getResourceAsStream("fillEmptyIntegerAction.json"));
    // when
    ActionTestWorkbench.test(row, actionRegistry, factory.create(action, parameters));
    // then
    Assert.assertEquals("25", row.get("0001"));
    Assert.assertEquals("David Bowie", row.get("0000"));
}