Examples with LocalStoreLocation org.talend.dataprep.api.dataset.location.LocalStoreLocation used on opensource projects

Example 1 with LocalStoreLocation

use of org.talend.dataprep.api.dataset.location.LocalStoreLocation in project data-prep by Talend.

the class DataSetLocatorService method getDataSetLocation.

/**
 * Return the dataset location from the content type and the connection parameters.
 *
 * @param contentType the dataset content type.
 * @param connectionParameters the connection parameters in json.
 * @return the dataset location.
 * @throws IOException if there's an error reading dataset location.
 */
public DataSetLocation getDataSetLocation(String contentType, InputStream connectionParameters) throws IOException {
    DataSetLocation location = null;
    // go through all dataset locator until one is able to retrieve the location
    for (DataSetLocator locator : locators) {
        if (locator.accept(contentType)) {
            location = locator.getLocation(connectionParameters);
            break;
        }
    }
    // local store location as fallback / default
    if (location == null) {
        location = new LocalStoreLocation();
    }
    LOG.debug("Location is {} for content type {}", location, contentType);
    return location;
}

Example 2 with LocalStoreLocation

use of org.talend.dataprep.api.dataset.location.LocalStoreLocation in project data-prep by Talend.

the class DataSetMetadataBuilderTest method createCompleteMetadata.

private DataSetMetadata createCompleteMetadata() {
    final Map<String, String> parameters = new HashMap<>(0);
    parameters.put("encoding", "UTF-8");
    final List<ColumnMetadata> columnsMetadata = new ArrayList<>(2);
    columnsMetadata.add(ColumnMetadata.Builder.column().id(0).name("id").type(INTEGER).build());
    columnsMetadata.add(ColumnMetadata.Builder.column().id(1).name("Name").type(STRING).build());
    final RowMetadata rowMetadata = new RowMetadata();
    rowMetadata.setColumns(columnsMetadata);
    final DataSetContent content = new DataSetContent();
    content.setNbRecords(1000);
    content.setLimit(1000L);
    content.setNbLinesInHeader(10);
    content.setNbLinesInFooter(10);
    content.setFormatFamilyId("formatGuess#csv");
    content.setMediaType("text/csv");
    content.setParameters(parameters);
    final Schema schemaParserResult = new Schema.Builder().draft(true).build();
    final DataSetMetadata metadata = new DataSetMetadata("18ba64c154d5", "Avengers stats", "Stan Lee", System.currentTimeMillis(), System.currentTimeMillis(), rowMetadata, "1.0");
    metadata.setLocation(new LocalStoreLocation());
    metadata.getGovernance().setCertificationStep(CERTIFIED);
    metadata.setSheetName("Sheet 1");
    metadata.setDraft(true);
    metadata.setContent(content);
    metadata.setEncoding("UTF-8");
    metadata.getLifecycle().contentIndexed(true);
    metadata.getLifecycle().qualityAnalyzed(true);
    metadata.getLifecycle().schemaAnalyzed(true);
    metadata.getLifecycle().setInProgress(true);
    metadata.getLifecycle().setImporting(true);
    metadata.setSchemaParserResult(schemaParserResult);
    metadata.setTag("MyTag");
    return metadata;
}

Example 3 with LocalStoreLocation

use of org.talend.dataprep.api.dataset.location.LocalStoreLocation in project data-prep by Talend.

the class DataSetJSONTest method testWrite1.

@Test
public void testWrite1() throws Exception {
    final ColumnMetadata.Builder columnBuilder = // 
    ColumnMetadata.Builder.column().id(// 
    5).name(// 
    "column1").type(// 
    Type.STRING).empty(// 
    0).invalid(// 
    10).valid(50);
    DataSetMetadata metadata = metadataBuilder.metadata().id("1234").name("name").author("author").created(0).row(columnBuilder).build();
    final DataSetContent content = metadata.getContent();
    content.addParameter(CSVFormatFamily.SEPARATOR_PARAMETER, ",");
    content.setFormatFamilyId(new CSVFormatFamily().getBeanId());
    content.setMediaType("text/csv");
    metadata.getLifecycle().qualityAnalyzed(true);
    metadata.getLifecycle().schemaAnalyzed(true);
    LocalStoreLocation location = new LocalStoreLocation();
    metadata.setLocation(location);
    StringWriter writer = new StringWriter();
    DataSet dataSet = new DataSet();
    dataSet.setMetadata(metadata);
    to(dataSet, writer);
    assertThat(writer.toString(), sameJSONAsFile(DataSetJSONTest.class.getResourceAsStream("test2.json")));
}

Example 4 with LocalStoreLocation

use of org.talend.dataprep.api.dataset.location.LocalStoreLocation in project data-prep by Talend.

the class ContentAnalysisTest method createCsvDataSet.

/**
 * Create a dataset out of the given metadata using the given string as source.
 *
 * @param metadata the dataset metadata.
 * @param source the source that points to a csv file within the classpath, relative to this class' package.
 */
private void createCsvDataSet(DataSetMetadata metadata, String source) {
    metadata.setLocation(new LocalStoreLocation());
    metadata.getContent().setFormatFamilyId(CSVFormatFamily.BEAN_ID);
    metadata.getContent().addParameter(CSVFormatFamily.SEPARATOR_PARAMETER, ",");
    dataSetMetadataRepository.save(metadata);
    contentStore.storeAsRaw(metadata, this.getClass().getResourceAsStream(source));
}