Search in sources :

Example 1 with SEMANTIC

use of org.talend.dataprep.quality.AnalyzerService.Analysis.SEMANTIC in project data-prep by Talend.

the class TransformationService method getSemanticDomains.

/**
 * Return the semantic domains for the given parameters.
 *
 * @param metadata the dataset metadata.
 * @param columnId the column id to analyze.
 * @param records the dataset records.
 * @return the semantic domains for the given parameters.
 * @throws IOException can happen...
 */
private List<SemanticDomain> getSemanticDomains(DataSetMetadata metadata, String columnId, InputStream records) throws IOException {
    // copy the column metadata and set the semantic domain forced flag to false to make sure the statistics adapter set all
    // available domains
    final ColumnMetadata columnMetadata = // 
    column().copy(// 
    metadata.getRowMetadata().getById(columnId)).semanticDomainForce(// 
    false).build();
    final Analyzer<Analyzers.Result> analyzer = analyzerService.build(columnMetadata, SEMANTIC);
    analyzer.init();
    try (final JsonParser parser = mapper.getFactory().createParser(new InputStreamReader(records, UTF_8))) {
        final DataSet dataSet = mapper.readerFor(DataSet.class).readValue(parser);
        dataSet.getRecords().map(// 
        r -> r.get(columnId)).forEach(analyzer::analyze);
        analyzer.end();
    }
    final List<Analyzers.Result> analyzerResult = analyzer.getResult();
    statisticsAdapter.adapt(singletonList(columnMetadata), analyzerResult);
    return columnMetadata.getSemanticDomains();
}
Also used : VolumeMetered(org.talend.dataprep.metrics.VolumeMetered) LocaleContextHolder(org.springframework.context.i18n.LocaleContextHolder) StringUtils(org.apache.commons.lang.StringUtils) ContentCacheKey(org.talend.dataprep.cache.ContentCacheKey) TdqCategories(org.talend.dataquality.semantic.broadcast.TdqCategories) Autowired(org.springframework.beans.factory.annotation.Autowired) ApiParam(io.swagger.annotations.ApiParam) PreviewParameters(org.talend.dataprep.transformation.preview.api.PreviewParameters) ExportFormatMessage(org.talend.dataprep.format.export.ExportFormatMessage) ActionContext(org.talend.dataprep.transformation.api.action.context.ActionContext) Collections.singletonList(java.util.Collections.singletonList) ScopeCategory(org.talend.dataprep.transformation.actions.category.ScopeCategory) Valid(javax.validation.Valid) SemanticDomain(org.talend.dataprep.api.dataset.statistics.SemanticDomain) BeanConversionService(org.talend.dataprep.conversions.BeanConversionService) GetPrepMetadataAsyncCondition(org.talend.dataprep.async.conditional.GetPrepMetadataAsyncCondition) TaskExecutor(org.springframework.core.task.TaskExecutor) DataSet(org.talend.dataprep.api.dataset.DataSet) ExportParametersUtil(org.talend.dataprep.api.export.ExportParametersUtil) StepDiff(org.talend.dataprep.api.preparation.StepDiff) PreparationDetailsGet(org.talend.dataprep.command.preparation.PreparationDetailsGet) HEAD(org.talend.dataprep.api.export.ExportParameters.SourceType.HEAD) APPLICATION_OCTET_STREAM_VALUE(org.springframework.http.MediaType.APPLICATION_OCTET_STREAM_VALUE) Resource(javax.annotation.Resource) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) JSON(org.talend.dataprep.transformation.format.JsonFormat.JSON) PreparationGetContentUrlGenerator(org.talend.dataprep.async.result.PreparationGetContentUrlGenerator) SecurityProxy(org.talend.dataprep.security.SecurityProxy) Stream(java.util.stream.Stream) Builder.column(org.talend.dataprep.api.dataset.ColumnMetadata.Builder.column) org.springframework.web.bind.annotation(org.springframework.web.bind.annotation) GZIPOutputStream(java.util.zip.GZIPOutputStream) DynamicType(org.talend.dataprep.transformation.api.action.dynamic.DynamicType) RunnableAction(org.talend.dataprep.transformation.actions.common.RunnableAction) Analyzers(org.talend.dataquality.common.inference.Analyzers) java.util(java.util) TransformationErrorCodes(org.talend.dataprep.exception.error.TransformationErrorCodes) GenericParameter(org.talend.dataprep.transformation.api.action.dynamic.GenericParameter) Configuration(org.talend.dataprep.transformation.api.transformer.configuration.Configuration) PreviewConfiguration(org.talend.dataprep.transformation.api.transformer.configuration.PreviewConfiguration) AnalyzerService(org.talend.dataprep.quality.AnalyzerService) TransformationMetadataCacheKey(org.talend.dataprep.cache.TransformationMetadataCacheKey) PrepMetadataExecutionIdGenerator(org.talend.dataprep.async.generator.PrepMetadataExecutionIdGenerator) PREPARATION_DOES_NOT_EXIST(org.talend.dataprep.exception.error.PreparationErrorCodes.PREPARATION_DOES_NOT_EXIST) Api(io.swagger.annotations.Api) Preparation(org.talend.dataprep.api.preparation.Preparation) ActionRegistry(org.talend.dataprep.transformation.pipeline.ActionRegistry) GetPrepContentAsyncCondition(org.talend.dataprep.async.conditional.GetPrepContentAsyncCondition) TransformationContext(org.talend.dataprep.transformation.api.action.context.TransformationContext) AggregationService(org.talend.dataprep.transformation.aggregation.AggregationService) NullOutputStream(org.apache.commons.io.output.NullOutputStream) StatisticsAdapter(org.talend.dataprep.dataset.StatisticsAdapter) DataSetGetMetadata(org.talend.dataprep.command.dataset.DataSetGetMetadata) Timed(org.talend.dataprep.metrics.Timed) ColumnMetadata(org.talend.dataprep.api.dataset.ColumnMetadata) DataSetGet(org.talend.dataprep.command.dataset.DataSetGet) AggregationResult(org.talend.dataprep.transformation.aggregation.api.AggregationResult) LoggerFactory(org.slf4j.LoggerFactory) Flag(org.talend.dataprep.api.dataset.row.Flag) SEMANTIC(org.talend.dataprep.quality.AnalyzerService.Analysis.SEMANTIC) ActionParser(org.talend.dataprep.transformation.api.action.ActionParser) CacheKeyGenerator(org.talend.dataprep.cache.CacheKeyGenerator) ApiOperation(io.swagger.annotations.ApiOperation) DataSetMetadata(org.talend.dataprep.api.dataset.DataSetMetadata) ExportParameters(org.talend.dataprep.api.export.ExportParameters) PrepMetadataGetContentUrlGenerator(org.talend.dataprep.async.result.PrepMetadataGetContentUrlGenerator) MediaType(org.springframework.http.MediaType) PublicAPI(org.talend.dataprep.security.PublicAPI) RequestMethod(org.springframework.web.bind.annotation.RequestMethod) Collectors(java.util.stream.Collectors) ContentCache(org.talend.dataprep.cache.ContentCache) UNEXPECTED_EXCEPTION(org.talend.dataprep.exception.error.TransformationErrorCodes.UNEXPECTED_EXCEPTION) TransformerFactory(org.talend.dataprep.transformation.api.transformer.TransformerFactory) CommonErrorCodes(org.talend.dataprep.exception.error.CommonErrorCodes) Analyzer(org.talend.dataquality.common.inference.Analyzer) ActionDefinition(org.talend.dataprep.api.action.ActionDefinition) PreparationExportStrategy(org.talend.dataprep.transformation.service.export.PreparationExportStrategy) RowMetadata(org.talend.dataprep.api.dataset.RowMetadata) ExportFormat(org.talend.dataprep.format.export.ExportFormat) TDPException(org.talend.dataprep.exception.TDPException) JsonErrorCodeDescription(org.talend.dataprep.exception.json.JsonErrorCodeDescription) ExceptionContext.build(org.talend.daikon.exception.ExceptionContext.build) ExportParametersExecutionIdGenerator(org.talend.dataprep.async.generator.ExportParametersExecutionIdGenerator) ExceptionContext(org.talend.daikon.exception.ExceptionContext) org.talend.dataprep.async(org.talend.dataprep.async) Suggestion(org.talend.dataprep.transformation.api.transformer.suggestion.Suggestion) SuggestionEngine(org.talend.dataprep.transformation.api.transformer.suggestion.SuggestionEngine) Logger(org.slf4j.Logger) LocaleContextHolder.getLocale(org.springframework.context.i18n.LocaleContextHolder.getLocale) JsonParser(com.fasterxml.jackson.core.JsonParser) UTF_8(java.nio.charset.StandardCharsets.UTF_8) Step(org.talend.dataprep.api.preparation.Step) APPLICATION_JSON_VALUE(org.springframework.http.MediaType.APPLICATION_JSON_VALUE) ApplicationContext(org.springframework.context.ApplicationContext) ActionForm(org.talend.dataprep.api.action.ActionForm) AggregationParameters(org.talend.dataprep.transformation.aggregation.api.AggregationParameters) java.io(java.io) TdqCategoriesFactory(org.talend.dataquality.semantic.broadcast.TdqCategoriesFactory) ColumnMetadata(org.talend.dataprep.api.dataset.ColumnMetadata) DataSet(org.talend.dataprep.api.dataset.DataSet) AggregationResult(org.talend.dataprep.transformation.aggregation.api.AggregationResult) JsonParser(com.fasterxml.jackson.core.JsonParser)

Example 2 with SEMANTIC

use of org.talend.dataprep.quality.AnalyzerService.Analysis.SEMANTIC in project data-prep by Talend.

the class DataSetService method getDataSetColumnSemanticCategories.

/**
 * Return the semantic types for a given dataset / column.
 *
 * @param datasetId the datasetId id.
 * @param columnId the column id.
 * @return the semantic types for a given dataset / column.
 */
@RequestMapping(value = "/datasets/{datasetId}/columns/{columnId}/types", method = GET)
@ApiOperation(value = "list the types of the wanted column", notes = "This list can be used by user to change the column type.")
@Timed
@PublicAPI
public List<SemanticDomain> getDataSetColumnSemanticCategories(@ApiParam(value = "The dataset id") @PathVariable String datasetId, @ApiParam(value = "The column id") @PathVariable String columnId) {
    LOG.debug("listing semantic categories for dataset #{} column #{}", datasetId, columnId);
    final DataSetMetadata metadata = dataSetMetadataRepository.get(datasetId);
    if (metadata == null) {
        throw new TDPException(DataSetErrorCodes.DATASET_DOES_NOT_EXIST, ExceptionContext.withBuilder().put("id", datasetId).build());
    } else {
        try (final Stream<DataSetRow> records = contentStore.stream(metadata)) {
            final ColumnMetadata columnMetadata = metadata.getRowMetadata().getById(columnId);
            final Analyzer<Analyzers.Result> analyzer = analyzerService.build(columnMetadata, SEMANTIC);
            analyzer.init();
            records.map(r -> r.get(columnId)).forEach(analyzer::analyze);
            analyzer.end();
            final List<Analyzers.Result> analyzerResult = analyzer.getResult();
            final StatisticsAdapter statisticsAdapter = new StatisticsAdapter(40);
            statisticsAdapter.adapt(singletonList(columnMetadata), analyzerResult);
            LOG.debug("found {} for dataset #{}, column #{}", columnMetadata.getSemanticDomains(), datasetId, columnId);
            return columnMetadata.getSemanticDomains();
        }
    }
}
Also used : TDPException(org.talend.dataprep.exception.TDPException) VolumeMetered(org.talend.dataprep.metrics.VolumeMetered) RequestParam(org.springframework.web.bind.annotation.RequestParam) ImportBuilder(org.talend.dataprep.api.dataset.Import.ImportBuilder) FormatFamilyFactory(org.talend.dataprep.schema.FormatFamilyFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) ApiParam(io.swagger.annotations.ApiParam) StringUtils(org.apache.commons.lang3.StringUtils) TEXT_PLAIN_VALUE(org.springframework.http.MediaType.TEXT_PLAIN_VALUE) SortAndOrderHelper.getDataSetMetadataComparator(org.talend.dataprep.util.SortAndOrderHelper.getDataSetMetadataComparator) Collections.singletonList(java.util.Collections.singletonList) SemanticDomain(org.talend.dataprep.api.dataset.statistics.SemanticDomain) BeanConversionService(org.talend.dataprep.conversions.BeanConversionService) PipedInputStream(java.io.PipedInputStream) DistributedLock(org.talend.dataprep.lock.DistributedLock) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) DataprepBundle.message(org.talend.dataprep.i18n.DataprepBundle.message) UserData(org.talend.dataprep.api.user.UserData) TaskExecutor(org.springframework.core.task.TaskExecutor) MAX_STORAGE_MAY_BE_EXCEEDED(org.talend.dataprep.exception.error.DataSetErrorCodes.MAX_STORAGE_MAY_BE_EXCEEDED) DataSet(org.talend.dataprep.api.dataset.DataSet) LocalStoreLocation(org.talend.dataprep.api.dataset.location.LocalStoreLocation) FormatFamily(org.talend.dataprep.schema.FormatFamily) Resource(javax.annotation.Resource) Set(java.util.Set) DatasetUpdatedEvent(org.talend.dataprep.dataset.event.DatasetUpdatedEvent) RestController(org.springframework.web.bind.annotation.RestController) QuotaService(org.talend.dataprep.dataset.store.QuotaService) Stream(java.util.stream.Stream) StreamSupport.stream(java.util.stream.StreamSupport.stream) FlagNames(org.talend.dataprep.api.dataset.row.FlagNames) UNEXPECTED_CONTENT(org.talend.dataprep.exception.error.CommonErrorCodes.UNEXPECTED_CONTENT) Analyzers(org.talend.dataquality.common.inference.Analyzers) DataSetLocatorService(org.talend.dataprep.api.dataset.location.locator.DataSetLocatorService) Callable(java.util.concurrent.Callable) Schema(org.talend.dataprep.schema.Schema) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) RequestBody(org.springframework.web.bind.annotation.RequestBody) DataSetLocationService(org.talend.dataprep.api.dataset.location.DataSetLocationService) AnalyzerService(org.talend.dataprep.quality.AnalyzerService) UserDataRepository(org.talend.dataprep.user.store.UserDataRepository) Markers(org.talend.dataprep.log.Markers) Api(io.swagger.annotations.Api) DraftValidator(org.talend.dataprep.schema.DraftValidator) HttpResponseContext(org.talend.dataprep.http.HttpResponseContext) Sort(org.talend.dataprep.util.SortAndOrderHelper.Sort) IOException(java.io.IOException) PipedOutputStream(java.io.PipedOutputStream) FormatAnalysis(org.talend.dataprep.dataset.service.analysis.synchronous.FormatAnalysis) ContentAnalysis(org.talend.dataprep.dataset.service.analysis.synchronous.ContentAnalysis) SchemaAnalysis(org.talend.dataprep.dataset.service.analysis.synchronous.SchemaAnalysis) HttpStatus(org.springframework.http.HttpStatus) FilterService(org.talend.dataprep.api.filter.FilterService) Marker(org.slf4j.Marker) NullOutputStream(org.apache.commons.io.output.NullOutputStream) StatisticsAdapter(org.talend.dataprep.dataset.StatisticsAdapter) Timed(org.talend.dataprep.metrics.Timed) ColumnMetadata(org.talend.dataprep.api.dataset.ColumnMetadata) PathVariable(org.springframework.web.bind.annotation.PathVariable) DataSetMetadataBuilder(org.talend.dataprep.dataset.DataSetMetadataBuilder) URLDecoder(java.net.URLDecoder) DataSetErrorCodes(org.talend.dataprep.exception.error.DataSetErrorCodes) PUT(org.springframework.web.bind.annotation.RequestMethod.PUT) LoggerFactory(org.slf4j.LoggerFactory) SEMANTIC(org.talend.dataprep.quality.AnalyzerService.Analysis.SEMANTIC) ApiOperation(io.swagger.annotations.ApiOperation) UNABLE_TO_CREATE_OR_UPDATE_DATASET(org.talend.dataprep.exception.error.DataSetErrorCodes.UNABLE_TO_CREATE_OR_UPDATE_DATASET) DataSetRow(org.talend.dataprep.api.dataset.row.DataSetRow) StrictlyBoundedInputStream(org.talend.dataprep.dataset.store.content.StrictlyBoundedInputStream) DataSetMetadata(org.talend.dataprep.api.dataset.DataSetMetadata) UNSUPPORTED_CONTENT(org.talend.dataprep.exception.error.DataSetErrorCodes.UNSUPPORTED_CONTENT) TimeToLive(org.talend.dataprep.cache.ContentCache.TimeToLive) Order(org.talend.dataprep.util.SortAndOrderHelper.Order) Collections.emptyList(java.util.Collections.emptyList) PublicAPI(org.talend.dataprep.security.PublicAPI) RequestMethod(org.springframework.web.bind.annotation.RequestMethod) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) ContentCache(org.talend.dataprep.cache.ContentCache) INVALID_DATASET_NAME(org.talend.dataprep.exception.error.DataSetErrorCodes.INVALID_DATASET_NAME) List(java.util.List) Optional(java.util.Optional) Analyzer(org.talend.dataquality.common.inference.Analyzer) RequestHeader(org.springframework.web.bind.annotation.RequestHeader) Pattern(java.util.regex.Pattern) Security(org.talend.dataprep.security.Security) Spliterator(java.util.Spliterator) RowMetadata(org.talend.dataprep.api.dataset.RowMetadata) ComponentProperties(org.talend.dataprep.parameters.jsonschema.ComponentProperties) TDPException(org.talend.dataprep.exception.TDPException) JsonErrorCodeDescription(org.talend.dataprep.exception.json.JsonErrorCodeDescription) RequestMapping(org.springframework.web.bind.annotation.RequestMapping) UNABLE_CREATE_DATASET(org.talend.dataprep.exception.error.DataSetErrorCodes.UNABLE_CREATE_DATASET) HashMap(java.util.HashMap) GET(org.springframework.web.bind.annotation.RequestMethod.GET) Import(org.talend.dataprep.api.dataset.Import) ExceptionContext.build(org.talend.daikon.exception.ExceptionContext.build) ExceptionContext(org.talend.daikon.exception.ExceptionContext) Charset(java.nio.charset.Charset) UpdateColumnParameters(org.talend.dataprep.dataset.service.api.UpdateColumnParameters) VersionService(org.talend.dataprep.api.service.info.VersionService) POST(org.springframework.web.bind.annotation.RequestMethod.POST) OutputStream(java.io.OutputStream) DataSetLocation(org.talend.dataprep.api.dataset.DataSetLocation) Logger(org.slf4j.Logger) LocaleContextHolder.getLocale(org.springframework.context.i18n.LocaleContextHolder.getLocale) UpdateDataSetCacheKey(org.talend.dataprep.dataset.service.cache.UpdateDataSetCacheKey) IOUtils(org.apache.commons.compress.utils.IOUtils) APPLICATION_JSON_VALUE(org.springframework.http.MediaType.APPLICATION_JSON_VALUE) ResponseBody(org.springframework.web.bind.annotation.ResponseBody) Certification(org.talend.dataprep.api.dataset.DataSetGovernance.Certification) EncodingSupport(org.talend.dataprep.configuration.EncodingSupport) Comparator(java.util.Comparator) InputStream(java.io.InputStream) StatisticsAdapter(org.talend.dataprep.dataset.StatisticsAdapter) ColumnMetadata(org.talend.dataprep.api.dataset.ColumnMetadata) DataSetMetadata(org.talend.dataprep.api.dataset.DataSetMetadata) DataSetRow(org.talend.dataprep.api.dataset.row.DataSetRow) Timed(org.talend.dataprep.metrics.Timed) ApiOperation(io.swagger.annotations.ApiOperation) PublicAPI(org.talend.dataprep.security.PublicAPI) RequestMapping(org.springframework.web.bind.annotation.RequestMapping)

Aggregations

Api (io.swagger.annotations.Api)2 ApiOperation (io.swagger.annotations.ApiOperation)2 ApiParam (io.swagger.annotations.ApiParam)2 Collections.singletonList (java.util.Collections.singletonList)2 Collectors (java.util.stream.Collectors)2 Stream (java.util.stream.Stream)2 Resource (javax.annotation.Resource)2 NullOutputStream (org.apache.commons.io.output.NullOutputStream)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2 Autowired (org.springframework.beans.factory.annotation.Autowired)2 LocaleContextHolder.getLocale (org.springframework.context.i18n.LocaleContextHolder.getLocale)2 TaskExecutor (org.springframework.core.task.TaskExecutor)2 APPLICATION_JSON_VALUE (org.springframework.http.MediaType.APPLICATION_JSON_VALUE)2 JsonParser (com.fasterxml.jackson.core.JsonParser)1 java.io (java.io)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 OutputStream (java.io.OutputStream)1 PipedInputStream (java.io.PipedInputStream)1