Examples with DataSetLocation org.talend.dataprep.api.dataset.DataSetLocation used on opensource projects

Example 1 with DataSetLocation

use of org.talend.dataprep.api.dataset.DataSetLocation in project data-prep by Talend.

the class DataSetService method getImportParameters.

@RequestMapping(value = "/datasets/imports/{import}/parameters", method = GET, produces = APPLICATION_JSON_VALUE)
@ApiOperation(value = "Get the import parameters", notes = "This list can be used by user to change dataset encoding.")
@Timed
@PublicAPI
public // ComponentProperties
Object getImportParameters(@PathVariable("import") final String importType) {
    DataSetLocation matchingDatasetLocation = locationsService.findLocation(importType);
    Object parametersToReturn;
    if (matchingDatasetLocation == null) {
        parametersToReturn = emptyList();
    } else {
        if (matchingDatasetLocation.isSchemaOriented()) {
            parametersToReturn = matchingDatasetLocation.getParametersAsSchema(getLocale());
        } else {
            parametersToReturn = matchingDatasetLocation.getParameters(getLocale());
        }
    }
    return parametersToReturn;
}

Example 2 with DataSetLocation

use of org.talend.dataprep.api.dataset.DataSetLocation in project data-prep by Talend.

the class DataSetService method getDataStoreParameters.

@RequestMapping(value = "/datasets/{id}/datastore/properties", method = GET)
@ApiOperation(value = "Get the dataset import parameters", notes = "This list can be used by user to change dataset encoding.")
@Timed
public // ComponentProperties
Object getDataStoreParameters(@PathVariable("id") final String dataSetId) {
    DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId);
    Object parametersToReturn = null;
    if (dataSetMetadata != null) {
        DataSetLocation matchingDatasetLocation = locationsService.findLocation(dataSetMetadata.getLocation().getLocationType());
        if (matchingDatasetLocation == null) {
            parametersToReturn = emptyList();
        } else {
            if (matchingDatasetLocation.isSchemaOriented()) {
                ComponentProperties parametersAsSchema = matchingDatasetLocation.getParametersAsSchema(getLocale());
                parametersAsSchema.setProperties(dataSetMetadata.getLocation().getParametersAsSchema(getLocale()).getProperties());
                parametersToReturn = parametersAsSchema;
            } else {
                parametersToReturn = matchingDatasetLocation.getParameters(getLocale());
            }
        }
    }
    return parametersToReturn;
}

Example 3 with DataSetLocation

use of org.talend.dataprep.api.dataset.DataSetLocation in project data-prep by Talend.

the class DataSetService method create.

/**
 * Creates a new data set and returns the new data set id as text in the response.
 *
 * @param name An optional name for the new data set (might be <code>null</code>).
 * @param size An optional size for the newly created data set.
 * @param contentType the request content type.
 * @param content The raw content of the data set (might be a CSV, XLS...) or the connection parameter in case of a
 * remote csv.
 * @return The new data id.
 * @see DataSetService#get(boolean, boolean, String, String)
 */
// @formatter:off
@RequestMapping(value = "/datasets", method = POST, produces = TEXT_PLAIN_VALUE)
@ApiOperation(value = "Create a data set", produces = TEXT_PLAIN_VALUE, notes = "Create a new data set based on content provided in POST body. For documentation purposes, body is typed as 'text/plain' but operation accepts binary content too. Returns the id of the newly created data set.")
@Timed
@VolumeMetered
public String create(@ApiParam(value = "User readable name of the data set (e.g. 'Finance Report 2015', 'Test Data Set').") @RequestParam(defaultValue = "") String name, @ApiParam(value = "An optional tag to be added in data set metadata once created.") @RequestParam(defaultValue = "") String tag, @ApiParam(value = "Size of the data set, in bytes.") @RequestParam(required = false) Long size, @RequestHeader(CONTENT_TYPE) String contentType, @ApiParam(value = "content") InputStream content) {
    // @formatter:on
    checkDataSetName(name);
    final String id = UUID.randomUUID().toString();
    final Marker marker = Markers.dataset(id);
    LOG.debug(marker, "Creating...");
    // sanity check
    if (size != null && size < 0) {
        LOG.warn("invalid size provided {}", size);
        throw new TDPException(UNEXPECTED_CONTENT, build().put("size", size));
    }
    // check that the name is not already taken
    checkIfNameIsAvailable(name);
    // get the location out of the content type and the request body
    final DataSetLocation location;
    try {
        location = datasetLocator.getDataSetLocation(contentType, content);
    } catch (IOException e) {
        throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_LOCATION, e);
    }
    DataSetMetadata dataSetMetadata = null;
    final TDPException hypotheticalException;
    try {
        // if the size is provided, let's check if the quota will not be exceeded
        if (size != null && size > 0) {
            quotaService.checkIfAddingSizeExceedsAvailableStorage(size);
        }
        dataSetMetadata = // 
        metadataBuilder.metadata().id(// 
        id).name(// 
        name).author(// 
        security.getUserId()).location(// 
        location).created(// 
        System.currentTimeMillis()).tag(// 
        tag).build();
        // Indicate data set is being imported
        dataSetMetadata.getLifecycle().setImporting(true);
        // Save data set content
        LOG.debug(marker, "Storing content...");
        final long maxDataSetSizeAllowed = getMaxDataSetSizeAllowed();
        final StrictlyBoundedInputStream sizeCalculator = new StrictlyBoundedInputStream(content, maxDataSetSizeAllowed);
        contentStore.storeAsRaw(dataSetMetadata, sizeCalculator);
        dataSetMetadata.setDataSetSize(sizeCalculator.getTotal());
        LOG.debug(marker, "Content stored.");
        // Create the new data set
        dataSetMetadataRepository.save(dataSetMetadata);
        LOG.debug(marker, "dataset metadata stored {}", dataSetMetadata);
        // Queue events (format analysis, content indexing for search...)
        analyzeDataSet(id, true, emptyList());
        LOG.debug(marker, "Created!");
        return id;
    } catch (StrictlyBoundedInputStream.InputStreamTooLargeException e) {
        hypotheticalException = new TDPException(MAX_STORAGE_MAY_BE_EXCEEDED, e, build().put("limit", e.getMaxSize()));
    } catch (TDPException e) {
        hypotheticalException = e;
    } catch (Exception e) {
        hypotheticalException = new TDPException(UNABLE_CREATE_DATASET, e);
    } finally {
        // because the client might still be writing the request content, closing the connexion right now
        // might end up in a 'connection reset' or a 'broken pipe' error in API.
        // 
        // So, let's read fully the request content before closing the connection.
        dataSetContentToNull(content);
    }
    dataSetMetadataRepository.remove(id);
    if (dataSetMetadata != null) {
        try {
            contentStore.delete(dataSetMetadata);
        } catch (Exception e) {
            LOG.error("Unable to delete uploaded data.", e);
        }
    }
    throw hypotheticalException;
}

Example 4 with DataSetLocation

use of org.talend.dataprep.api.dataset.DataSetLocation in project data-prep by Talend.

the class DataSetLocatorService method getDataSetLocation.

/**
 * Return the dataset location from the content type and the connection parameters.
 *
 * @param contentType the dataset content type.
 * @param connectionParameters the connection parameters in json.
 * @return the dataset location.
 * @throws IOException if there's an error reading dataset location.
 */
public DataSetLocation getDataSetLocation(String contentType, InputStream connectionParameters) throws IOException {
    DataSetLocation location = null;
    // go through all dataset locator until one is able to retrieve the location
    for (DataSetLocator locator : locators) {
        if (locator.accept(contentType)) {
            location = locator.getLocation(connectionParameters);
            break;
        }
    }
    // local store location as fallback / default
    if (location == null) {
        location = new LocalStoreLocation();
    }
    LOG.debug("Location is {} for content type {}", location, contentType);
    return location;
}