Examples with DistributedLock org.talend.dataprep.lock.DistributedLock used on opensource projects

Example 6 with DistributedLock

use of org.talend.dataprep.lock.DistributedLock in project data-prep by Talend.

the class ContentAnalysis method analyze.

/**
 * @see DataSetAnalyzer#analyze(String)
 */
@Override
public void analyze(String dataSetId) {
    // defensive programming
    if (StringUtils.isEmpty(dataSetId)) {
        throw new IllegalArgumentException("Data set id cannot be null or empty.");
    }
    DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId);
    datasetLock.lock();
    try {
        DataSetMetadata metadata = repository.get(dataSetId);
        if (metadata != null) {
            LOG.info("Indexing content of data set #{}...", metadata.getId());
            updateHeaderAndFooter(metadata);
            updateLimit(metadata);
            metadata.getLifecycle().contentIndexed(true);
            repository.save(metadata);
            LOG.info("Indexed content of data set #{}.", dataSetId);
        } else {
            // $NON-NLS-1$
            LOG.info("Data set #{} no longer exists.", dataSetId);
        }
    } finally {
        datasetLock.unlock();
    }
}

Example 7 with DistributedLock

use of org.talend.dataprep.lock.DistributedLock in project data-prep by Talend.

the class DataSetService method updateDataSet.

/**
 * Updates a data set metadata. If no data set exists for given id, a {@link TDPException} is thrown.
 *
 * @param dataSetId The id of data set to be updated.
 * @param dataSetMetadata The new content for the data set. If empty, existing content will <b>not</b> be replaced.
 * For delete operation, look at {@link #delete(String)}.
 */
@RequestMapping(value = "/datasets/{id}", method = PUT)
@ApiOperation(value = "Update a data set metadata by id", notes = "Update a data set metadata according to the content of the PUT body. Id should be a UUID returned by the list operation. Not valid or non existing data set id return an error response.")
@Timed
public void updateDataSet(@PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the data set to update") String dataSetId, @RequestBody DataSetMetadata dataSetMetadata) {
    if (dataSetMetadata != null && dataSetMetadata.getName() != null) {
        checkDataSetName(dataSetMetadata.getName());
    }
    final DistributedLock lock = dataSetMetadataRepository.createDatasetMetadataLock(dataSetId);
    lock.lock();
    try {
        DataSetMetadata metadataForUpdate = dataSetMetadataRepository.get(dataSetId);
        if (metadataForUpdate == null) {
            // No need to silently create the data set metadata: associated content will most likely not exist.
            throw new TDPException(DataSetErrorCodes.DATASET_DOES_NOT_EXIST, build().put("id", dataSetId));
        }
        LOG.debug("updateDataSet: {}", dataSetMetadata);
        publisher.publishEvent(new DatasetUpdatedEvent(dataSetMetadata));
        // 
        // Only part of the metadata can be updated, so the original dataset metadata is loaded and updated
        // 
        DataSetMetadata original = metadataBuilder.metadata().copy(metadataForUpdate).build();
        try {
            // update the name
            metadataForUpdate.setName(dataSetMetadata.getName());
            // update the sheet content (in case of a multi-sheet excel file)
            if (metadataForUpdate.getSchemaParserResult() != null) {
                Optional<Schema.SheetContent> sheetContentFound = metadataForUpdate.getSchemaParserResult().getSheetContents().stream().filter(sheetContent -> dataSetMetadata.getSheetName().equals(sheetContent.getName())).findFirst();
                if (sheetContentFound.isPresent()) {
                    List<ColumnMetadata> columnMetadatas = sheetContentFound.get().getColumnMetadatas();
                    if (metadataForUpdate.getRowMetadata() == null) {
                        metadataForUpdate.setRowMetadata(new RowMetadata(emptyList()));
                    }
                    metadataForUpdate.getRowMetadata().setColumns(columnMetadatas);
                }
                metadataForUpdate.setSheetName(dataSetMetadata.getSheetName());
                metadataForUpdate.setSchemaParserResult(null);
            }
            // Location updates
            metadataForUpdate.setLocation(dataSetMetadata.getLocation());
            // update parameters & encoding (so that user can change import parameters for CSV)
            metadataForUpdate.getContent().setParameters(dataSetMetadata.getContent().getParameters());
            metadataForUpdate.setEncoding(dataSetMetadata.getEncoding());
            // update limit
            final Optional<Long> newLimit = dataSetMetadata.getContent().getLimit();
            newLimit.ifPresent(limit -> metadataForUpdate.getContent().setLimit(limit));
            // Validate that the new data set metadata and removes the draft status
            final String formatFamilyId = dataSetMetadata.getContent().getFormatFamilyId();
            if (formatFamilyFactory.hasFormatFamily(formatFamilyId)) {
                FormatFamily format = formatFamilyFactory.getFormatFamily(formatFamilyId);
                try {
                    DraftValidator draftValidator = format.getDraftValidator();
                    DraftValidator.Result result = draftValidator.validate(dataSetMetadata);
                    if (result.isDraft()) {
                        // This is not an exception case: data set may remain a draft after update (although rather
                        // unusual)
                        LOG.warn("Data set #{} is still a draft after update.", dataSetId);
                        return;
                    }
                    // Data set metadata to update is no longer a draft
                    metadataForUpdate.setDraft(false);
                } catch (UnsupportedOperationException e) {
                // no need to validate draft here
                }
            }
            // update schema
            formatAnalyzer.update(original, metadataForUpdate);
            // save the result
            dataSetMetadataRepository.save(metadataForUpdate);
            // all good mate!! so send that to jms
            // Asks for a in depth schema analysis (for column type information).
            analyzeDataSet(dataSetId, true, singletonList(FormatAnalysis.class));
        } catch (TDPException e) {
            throw e;
        } catch (Exception e) {
            throw new TDPException(UNABLE_TO_CREATE_OR_UPDATE_DATASET, e);
        }
    } finally {
        lock.unlock();
    }
}

Example 8 with DistributedLock

use of org.talend.dataprep.lock.DistributedLock in project data-prep by Talend.

the class DataSetService method updateDatasetColumn.

/**
 * Update the column of the data set and computes the
 *
 * @param dataSetId the dataset id.
 * @param columnId the column id.
 * @param parameters the new type and domain.
 */
@RequestMapping(value = "/datasets/{datasetId}/column/{columnId}", method = POST)
@ApiOperation(value = "Update a column type and/or domain")
@Timed
public void updateDatasetColumn(@PathVariable(value = "datasetId") @ApiParam(name = "datasetId", value = "Id of the dataset") final String dataSetId, @PathVariable(value = "columnId") @ApiParam(name = "columnId", value = "Id of the column") final String columnId, @RequestBody final UpdateColumnParameters parameters) {
    final DistributedLock lock = dataSetMetadataRepository.createDatasetMetadataLock(dataSetId);
    lock.lock();
    try {
        // check that dataset exists
        final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId);
        if (dataSetMetadata == null) {
            throw new TDPException(DataSetErrorCodes.DATASET_DOES_NOT_EXIST, build().put("id", dataSetId));
        }
        LOG.debug("update dataset column for #{} with type {} and/or domain {}", dataSetId, parameters.getType(), parameters.getDomain());
        // get the column
        final ColumnMetadata column = dataSetMetadata.getRowMetadata().getById(columnId);
        if (column == null) {
            throw new // 
            TDPException(// 
            DataSetErrorCodes.COLUMN_DOES_NOT_EXIST, // 
            build().put("id", // 
            dataSetId).put("columnid", columnId));
        }
        // update type/domain
        if (parameters.getType() != null) {
            column.setType(parameters.getType());
        }
        if (parameters.getDomain() != null) {
            // erase domain to let only type
            if (parameters.getDomain().isEmpty()) {
                column.setDomain("");
                column.setDomainLabel("");
                column.setDomainFrequency(0);
            } else // change domain
            {
                final SemanticDomain semanticDomain = column.getSemanticDomains().stream().filter(// 
                dom -> StringUtils.equals(dom.getId(), parameters.getDomain())).findFirst().orElse(null);
                if (semanticDomain != null) {
                    column.setDomain(semanticDomain.getId());
                    column.setDomainLabel(semanticDomain.getLabel());
                    column.setDomainFrequency(semanticDomain.getScore());
                }
            }
        }
        // save
        dataSetMetadataRepository.save(dataSetMetadata);
        // analyze the updated dataset (not all analysis are performed)
        analyzeDataSet(// 
        dataSetId, // 
        false, asList(ContentAnalysis.class, FormatAnalysis.class, SchemaAnalysis.class));
    } finally {
        lock.unlock();
    }
}

Example 9 with DistributedLock

use of org.talend.dataprep.lock.DistributedLock in project data-prep by Talend.

the class BackgroundAnalysis method analyze.

/**
 * @see DataSetAnalyzer#analyze
 */
public void analyze(String dataSetId) {
    if (StringUtils.isEmpty(dataSetId)) {
        throw new IllegalArgumentException("Data set id cannot be null or empty.");
    }
    LOGGER.debug("Statistics analysis starts for {}", dataSetId);
    DataSetMetadata metadata = repository.get(dataSetId);
    if (metadata != null) {
        if (!metadata.getLifecycle().schemaAnalyzed()) {
            LOGGER.debug("Dataset {}, schema information must be computed before quality analysis can be performed, ignoring message", metadata.getId());
            // no acknowledge to allow re-poll.
            return;
        }
        final List<ColumnMetadata> columns = metadata.getRowMetadata().getColumns();
        if (columns.isEmpty()) {
            LOGGER.debug("Skip statistics of {} (no column information).", metadata.getId());
        } else {
            // base analysis
            try (final Stream<DataSetRow> stream = store.stream(metadata)) {
                try (Analyzer<Analyzers.Result> analyzer = analyzerService.schemaAnalysis(columns)) {
                    computeStatistics(analyzer, columns, stream);
                    LOGGER.debug("Base statistics analysis done for{}", dataSetId);
                    // Save base analysis
                    saveAnalyzerResults(dataSetId, analyzer);
                }
            } catch (Exception e) {
                LOGGER.warn("Base statistics analysis, dataset {} generates an error", dataSetId, e);
                throw new TDPException(UNABLE_TO_ANALYZE_DATASET_QUALITY, e);
            }
            // advanced analysis
            try (final Stream<DataSetRow> stream = store.stream(metadata)) {
                try (Analyzer<Analyzers.Result> analyzer = analyzerService.full(columns)) {
                    computeStatistics(analyzer, columns, stream);
                    updateNbRecords(metadata, analyzer.getResult());
                    LOGGER.debug("Advanced statistics analysis done for{}", dataSetId);
                    // Save advanced analysis
                    saveAnalyzerResults(dataSetId, analyzer);
                }
            } catch (Exception e) {
                LOGGER.warn("Advanced statistics analysis, dataset {} generates an error", dataSetId, e);
                throw new TDPException(UNABLE_TO_ANALYZE_DATASET_QUALITY, e);
            }
            // Tag data set quality: now analyzed
            DistributedLock datasetLock = repository.createDatasetMetadataLock(metadata.getId());
            try {
                datasetLock.lock();
                final DataSetMetadata dataSetMetadata = repository.get(dataSetId);
                if (dataSetMetadata != null) {
                    dataSetMetadata.getLifecycle().qualityAnalyzed(true);
                    repository.save(metadata);
                }
            } finally {
                datasetLock.unlock();
            }
            LOGGER.info("Statistics analysis done for {}", dataSetId);
        }
    } else {
        LOGGER.info("Unable to analyze quality of data set #{}: seems to be removed.", dataSetId);
    }
}

Example 10 with DistributedLock

use of org.talend.dataprep.lock.DistributedLock in project data-prep by Talend.

the class BackgroundAnalysis method saveAnalyzerResults.

private void saveAnalyzerResults(String id, Analyzer<Analyzers.Result> analyzer) {
    DistributedLock datasetLock = repository.createDatasetMetadataLock(id);
    try {
        datasetLock.lock();
        final DataSetMetadata dataSetMetadata = repository.get(id);
        if (dataSetMetadata != null) {
            adapter.adapt(dataSetMetadata.getRowMetadata().getColumns(), analyzer.getResult());
            repository.save(dataSetMetadata);
        }
    } finally {
        datasetLock.unlock();
    }
}