Examples with Timed org.talend.dataprep.metrics.Timed used on opensource projects

Example 11 with Timed

use of org.talend.dataprep.metrics.Timed in project data-prep by Talend.

the class DataSetService method preview.

/**
 * Returns preview of the the data set content for given id (first 100 rows). Service might return
 * {@link org.apache.http.HttpStatus#SC_ACCEPTED} if the data set exists but analysis is not yet fully
 * completed so content is not yet ready to be served.
 *
 * @param metadata If <code>true</code>, includes data set metadata information.
 * @param sheetName the sheet name to preview
 * @param dataSetId A data set id.
 */
@RequestMapping(value = "/datasets/{id}/preview", method = RequestMethod.GET)
@ApiOperation(value = "Get a data preview set by id", notes = "Get a data set preview content based on provided id. Not valid or non existing data set id returns empty content. Data set not in drat status will return a redirect 301")
@Timed
@ResponseBody
public DataSet preview(@RequestParam(defaultValue = "true") @ApiParam(name = "metadata", value = "Include metadata information in the response") boolean metadata, @RequestParam(defaultValue = "") @ApiParam(name = "sheetName", value = "Sheet name to preview") String sheetName, @PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the requested data set") String dataSetId) {
    DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId);
    if (dataSetMetadata == null) {
        HttpResponseContext.status(HttpStatus.NO_CONTENT);
        // No data set, returns empty content.
        return DataSet.empty();
    }
    if (!dataSetMetadata.isDraft()) {
        // Moved to get data set content operation
        HttpResponseContext.status(HttpStatus.MOVED_PERMANENTLY);
        HttpResponseContext.header("Location", "/datasets/" + dataSetId + "/content");
        // dataset not anymore a draft so preview doesn't make sense.
        return DataSet.empty();
    }
    if (StringUtils.isNotEmpty(sheetName)) {
        dataSetMetadata.setSheetName(sheetName);
    }
    // take care of previous data without schema parser result
    if (dataSetMetadata.getSchemaParserResult() != null) {
        // sheet not yet set correctly so use the first one
        if (StringUtils.isEmpty(dataSetMetadata.getSheetName())) {
            String theSheetName = dataSetMetadata.getSchemaParserResult().getSheetContents().get(0).getName();
            LOG.debug("preview for dataSetMetadata: {} with sheetName: {}", dataSetId, theSheetName);
            dataSetMetadata.setSheetName(theSheetName);
        }
        String theSheetName = dataSetMetadata.getSheetName();
        Optional<Schema.SheetContent> sheetContentFound = dataSetMetadata.getSchemaParserResult().getSheetContents().stream().filter(sheetContent -> theSheetName.equals(sheetContent.getName())).findFirst();
        if (!sheetContentFound.isPresent()) {
            HttpResponseContext.status(HttpStatus.NO_CONTENT);
            // No sheet found, returns empty content.
            return DataSet.empty();
        }
        List<ColumnMetadata> columnMetadatas = sheetContentFound.get().getColumnMetadatas();
        if (dataSetMetadata.getRowMetadata() == null) {
            dataSetMetadata.setRowMetadata(new RowMetadata(emptyList()));
        }
        dataSetMetadata.getRowMetadata().setColumns(columnMetadatas);
    } else {
        LOG.warn("dataset#{} has draft status but any SchemaParserResult");
    }
    // Build the result
    DataSet dataSet = new DataSet();
    if (metadata) {
        dataSet.setMetadata(conversionService.convert(dataSetMetadata, UserDataSetMetadata.class));
    }
    dataSet.setRecords(contentStore.stream(dataSetMetadata).limit(100));
    return dataSet;
}

Example 12 with Timed

use of org.talend.dataprep.metrics.Timed in project data-prep by Talend.

the class DataSetService method copy.

/**
 * Copy this dataset to a new one and returns the new data set id as text in the response.
 *
 * @param copyName the name of the copy
 * @return The new data id.
 */
@RequestMapping(value = "/datasets/{id}/copy", method = POST, produces = TEXT_PLAIN_VALUE)
@ApiOperation(value = "Copy a data set", produces = TEXT_PLAIN_VALUE, notes = "Copy a new data set based on the given id. Returns the id of the newly created data set.")
@Timed
public String copy(@PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the data set to clone") String dataSetId, @ApiParam(value = "The name of the cloned dataset.") @RequestParam(required = false) String copyName) throws IOException {
    if (copyName != null) {
        checkDataSetName(copyName);
    }
    HttpResponseContext.contentType(TEXT_PLAIN_VALUE);
    DataSetMetadata original = dataSetMetadataRepository.get(dataSetId);
    if (original == null) {
        return StringUtils.EMPTY;
    }
    // use a default name if empty (original name + " Copy" )
    final String newName;
    if (StringUtils.isBlank(copyName)) {
        newName = message("dataset.copy.newname", original.getName());
    } else {
        newName = copyName;
    }
    final DistributedLock lock = dataSetMetadataRepository.createDatasetMetadataLock(dataSetId);
    try {
        // lock to ensure any asynchronous analysis is completed.
        lock.lock();
        // check that the name is not already taken
        checkIfNameIsAvailable(newName);
        // check that there's enough space
        final long maxDataSetSizeAllowed = getMaxDataSetSizeAllowed();
        if (maxDataSetSizeAllowed < original.getDataSetSize()) {
            throw new TDPException(MAX_STORAGE_MAY_BE_EXCEEDED, build().put("limit", maxDataSetSizeAllowed));
        }
        // Create copy (based on original data set metadata)
        final String newId = UUID.randomUUID().toString();
        final Marker marker = Markers.dataset(newId);
        LOG.debug(marker, "Cloning...");
        DataSetMetadata target = // 
        metadataBuilder.metadata().copy(// 
        original).id(// 
        newId).name(// 
        newName).author(// 
        security.getUserId()).location(// 
        original.getLocation()).created(// 
        System.currentTimeMillis()).build();
        // Save data set content
        LOG.debug(marker, "Storing content...");
        try (InputStream content = contentStore.getAsRaw(original)) {
            contentStore.storeAsRaw(target, content);
        }
        LOG.debug(marker, "Content stored.");
        // Create the new data set
        dataSetMetadataRepository.save(target);
        LOG.info(marker, "Copy done --> {}", newId);
        return newId;
    } finally {
        lock.unlock();
    }
}

Example 13 with Timed

use of org.talend.dataprep.metrics.Timed in project data-prep by Talend.

the class DataSetService method delete.

/**
 * Deletes a data set with provided id.
 *
 * @param dataSetId A data set id. If data set id is unknown, no exception nor status code to indicate this is set.
 */
@RequestMapping(value = "/datasets/{id}", method = RequestMethod.DELETE)
@ApiOperation(value = "Delete a data set by id", notes = "Delete a data set content based on provided id. Id should be a UUID returned by the list operation. Not valid or non existing data set id returns empty content.")
@Timed
public void delete(@PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the data set to delete") String dataSetId) {
    DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId);
    final DistributedLock lock = dataSetMetadataRepository.createDatasetMetadataLock(dataSetId);
    try {
        lock.lock();
        if (metadata != null) {
            // first remove the metadata as there may be additional check
            dataSetMetadataRepository.remove(dataSetId);
            contentStore.delete(metadata);
        } else {
            HttpResponseContext.status(HttpStatus.NOT_FOUND);
        }
    } finally {
        lock.unlock();
    }
}

Example 14 with Timed

use of org.talend.dataprep.metrics.Timed in project data-prep by Talend.

the class ResourceLoaderContentCache method evictMatch.

@Timed
@Override
public void evictMatch(ContentCacheKey key) {
    try {
        final DeletableResource[] resources = resolver.getResources("/cache/" + key.getPrefix() + "**");
        final Predicate<String> matcher = key.getMatcher();
        stream(resources).filter(r -> matcher.test(r.getFilename())).forEach(r -> {
            try {
                r.delete();
            } catch (IOException e) {
                throw new TDPException(CommonErrorCodes.UNEXPECTED_EXCEPTION, e);
            }
        });
    } catch (IOException e) {
        throw new TDPException(CommonErrorCodes.UNEXPECTED_EXCEPTION, e);
    }
}

Example 15 with Timed

use of org.talend.dataprep.metrics.Timed in project data-prep by Talend.

the class DataSetAPI method listSummary.

@RequestMapping(value = "/api/datasets/summary", method = GET, produces = APPLICATION_JSON_VALUE)
@ApiOperation(value = "List data sets summary.", produces = APPLICATION_JSON_VALUE, notes = "Returns a list of data sets summary the user can use.")
@Timed
public Callable<Stream<EnrichedDataSetMetadata>> listSummary(@ApiParam(value = "Sort key (by name or date), defaults to 'date'.") @RequestParam(defaultValue = "creationDate") Sort sort, @ApiParam(value = "Order for sort key (desc or asc), defaults to 'desc'.") @RequestParam(defaultValue = "desc") Order order, @ApiParam(value = "Filter on name containing the specified name") @RequestParam(defaultValue = "") String name, @ApiParam(value = "Filter on certified data sets") @RequestParam(defaultValue = "false") boolean certified, @ApiParam(value = "Filter on favorite data sets") @RequestParam(defaultValue = "false") boolean favorite, @ApiParam(value = "Filter on recent data sets") @RequestParam(defaultValue = "false") boolean limit) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Listing datasets summary (pool: {})...", getConnectionStats());
    }
    return () -> {
        GenericCommand<InputStream> listDataSets = getCommand(DataSetList.class, sort, order, name, certified, favorite, limit);
        return // 
        Flux.from(CommandHelper.toPublisher(UserDataSetMetadata.class, mapper, listDataSets)).map(m -> {
            LOG.debug("found dataset {} in the summary list" + m.getName());
            // Add the related preparations list to the given dataset metadata.
            final PreparationSearchByDataSetId getPreparations = getCommand(PreparationSearchByDataSetId.class, m.getId());
            return // 
            Flux.from(CommandHelper.toPublisher(Preparation.class, mapper, getPreparations)).collectList().map(preparations -> {
                final List<Preparation> list = // 
                preparations.stream().filter(// 
                p -> p.getSteps() != null).collect(Collectors.toList());
                return new EnrichedDataSetMetadata(m, list);
            }).block();
        }).toStream(1);
    };
}