Examples with Connector com.thinkbiganalytics.kylo.catalog.rest.model.Connector used on opensource projects

Example 6 with Connector

use of com.thinkbiganalytics.kylo.catalog.rest.model.Connector in project kylo by Teradata.

the class DataSourceProvider method toDataSource.

/**
 * Converts a feed data source to a REST data source.
 */
@Nullable
@SuppressWarnings("squid:S3655")
private DataSource toDataSource(@Nullable final Datasource feedDataSource, @Nonnull final DatasourceModelTransform.Level level) {
    // Transform to metadata data source
    final com.thinkbiganalytics.metadata.rest.model.data.Datasource metadataDataSource;
    if (feedDataSource != null) {
        metadataDataSource = feedDataSourceTransform.toDatasource(feedDataSource, level);
    } else {
        return null;
    }
    // Add properties to data source
    final DataSource dataSource = new DataSource();
    dataSource.setId(metadataDataSource.getId());
    dataSource.setTitle(metadataDataSource.getName());
    // Set properties based on type
    final Connector connector = new Connector();
    final DefaultDataSetTemplate template;
    if (metadataDataSource instanceof JdbcDatasource && getJdbcConnectorId().isPresent()) {
        connector.setId(getJdbcConnectorId().get());
        template = createTemplate((JdbcDatasource) metadataDataSource);
    } else {
        return null;
    }
    dataSource.setConnector(connector);
    dataSource.setTemplate(template);
    return dataSource;
}

Example 7 with Connector

use of com.thinkbiganalytics.kylo.catalog.rest.model.Connector in project kylo by Teradata.

the class DataSourceProvider method findDataSource.

/**
 * Gets the connector with the specified id.
 */
@Nonnull
@SuppressWarnings("squid:S2259")
public Optional<DataSource> findDataSource(@Nonnull final String id, final boolean includeCredentials) {
    return metadataService.read(() -> {
        // Find the data source
        DataSource dataSource = metadataProvider.find(metadataProvider.resolveId(id)).map(modelTransform.dataSourceToRestModel()).orElse(null);
        if (dataSource != null) {
            dataSource = new DataSource(dataSource);
        } else {
            try {
                final Datasource feedDataSource = feedDataSourceProvider.getDatasource(feedDataSourceProvider.resolve(id));
                DatasourceModelTransform.Level level = DatasourceModelTransform.Level.FULL;
                if (includeCredentials) {
                    level = DatasourceModelTransform.Level.ADMIN;
                }
                dataSource = toDataSource(feedDataSource, level);
            } catch (final IllegalArgumentException e) {
                log.debug("Failed to resolve data source {}: {}", id, e, e);
            }
        }
        // Set connector
        final Optional<Connector> connector = Optional.ofNullable(dataSource).map(DataSource::getConnector).map(Connector::getId).map(connectorProvider::resolveId).flatMap(connectorProvider::find).map(modelTransform.connectorToRestModel());
        if (connector.isPresent()) {
            dataSource.setConnector(connector.get());
            return Optional.of(dataSource);
        } else {
            if (dataSource != null) {
                log.error("Unable to find connector for data source: {}", dataSource);
            }
            return Optional.empty();
        }
    });
}

Example 8 with Connector

use of com.thinkbiganalytics.kylo.catalog.rest.model.Connector in project kylo by Teradata.

the class KyloCatalogReaderUtil method toKyloCatalogRequest.

public static KyloCatalogReadRequest toKyloCatalogRequest(PreviewDataSetRequest previewRequest) {
    DataSource dataSource = previewRequest.getDataSource();
    Connector connector = dataSource.getConnector();
    // merge template
    DataSetTemplate dataSetTemplate = DataSourceUtil.mergeTemplates(dataSource);
    // get data out of the dataset template
    List<String> jars = dataSetTemplate.getJars();
    List<String> paths = dataSetTemplate.getPaths();
    List<String> files = dataSetTemplate.getFiles();
    String format = dataSetTemplate.getFormat();
    Map<String, String> options = dataSetTemplate.getOptions();
    if (options == null) {
        options = new HashMap<>();
    }
    // parse the SchemaParser if it exists and add options and update the format
    if (previewRequest.getSchemaParser() != null) {
        SchemaParserDescriptor schemaParser = previewRequest.getSchemaParser();
        Map<String, String> sparkOptions = schemaParser.getProperties().stream().collect(Collectors.toMap(p -> p.getAdditionalProperties().stream().filter(labelValue -> "spark.option".equalsIgnoreCase(labelValue.getLabel())).map(labelValue -> labelValue.getValue()).findFirst().orElse(""), p -> p.getValue()));
        // remove any options that produced an empty key
        sparkOptions.remove("");
        // supplied options by the schema parse take precedence over the template options
        options.putAll(sparkOptions);
        format = schemaParser.getSparkFormat();
    }
    // add in additional preview options
    if (previewRequest.getProperties() != null && !previewRequest.getProperties().isEmpty()) {
        options.putAll(previewRequest.getProperties());
    }
    KyloCatalogReadRequest request = new KyloCatalogReadRequest();
    request.setFiles(files);
    request.setJars(jars);
    request.setFormat(format);
    request.setOptions(options);
    if (previewRequest.getPreviewItem() != null && previewRequest.isAddPreviewItemToPath()) {
        request.addPath(previewRequest.getPreviewItem());
    }
    PageSpec pageSpec = previewRequest.getPageSpec();
    if (pageSpec == null) {
        pageSpec = new PageSpec();
    }
    request.setPageSpec(pageSpec);
    return request;
}

Example 9 with Connector

use of com.thinkbiganalytics.kylo.catalog.rest.model.Connector in project kylo by Teradata.

the class DataSourceIT method testListFilesS3.

/**
 * Verify listing files from the Amazon S3 connector.
 */
@Test
public void testListFilesS3() {
    Assume.assumeNotNull(awsAccessKeyId, awsSecretAccessKey);
    // Create an S3 data source
    final Connector connector = new Connector();
    connector.setId("amazon-s3");
    final DefaultDataSetTemplate template = new DefaultDataSetTemplate();
    template.setOptions(new HashMap<>());
    template.getOptions().put("spark.hadoop.fs.s3a.access.key", awsAccessKeyId);
    template.getOptions().put("spark.hadoop.fs.s3a.secret.key", awsSecretAccessKey);
    final DataSource request = new DataSource();
    request.setConnector(connector);
    request.setTemplate(template);
    request.setTitle("test list files s3");
    final DataSource dataSource = given(DataSourceController.BASE).when().body(request).post().then().statusCode(200).extract().as(DataSource.class);
    // Test listing buckets
    final List<DataSetFile> buckets = given(DataSourceController.BASE).when().pathParam("id", dataSource.getId()).queryParam("path", "s3a:/").get("{id}/files").then().statusCode(200).extract().as(DataSetFileList.class);
    Assert.assertThat(buckets, CoreMatchers.hasItem(new CustomMatcher<DataSetFile>("DataSetFile name=thinkbig.greg directory=true") {

        @Override
        public boolean matches(final Object item) {
            return (item instanceof DataSetFile) && Objects.equals("thinkbig.greg", ((DataSetFile) item).getName()) && Objects.equals("s3a://thinkbig.greg/", ((DataSetFile) item).getPath()) && ((DataSetFile) item).isDirectory();
        }
    }));
    // Test listing files
    final List<DataSetFile> files = given(DataSourceController.BASE).when().pathParam("id", dataSource.getId()).queryParam("path", "s3a://thinkbig.greg/").get("{id}/files").then().statusCode(200).extract().as(DataSetFileList.class);
    Assert.assertThat(files, CoreMatchers.hasItem(new CustomMatcher<DataSetFile>("DataSetFile name=userdata1.csv directory=false") {

        @Override
        public boolean matches(Object item) {
            return (item instanceof DataSetFile) && Objects.equals("userdata1.csv", ((DataSetFile) item).getName()) && Objects.equals("s3a://thinkbig.greg/userdata1.csv", ((DataSetFile) item).getPath()) && !((DataSetFile) item).isDirectory();
        }
    }));
}

Example 10 with Connector

use of com.thinkbiganalytics.kylo.catalog.rest.model.Connector in project kylo by Teradata.

the class DataSourceIT method testListFilesAzureNative.

/**
 * Verify listing files from the Azure Storage connector.
 */
@Test
public void testListFilesAzureNative() {
    Assume.assumeNotNull(azureAccountKey);
    // Create an Azure data source
    final Connector connector = new Connector();
    connector.setId("azure-storage");
    final DefaultDataSetTemplate template = new DefaultDataSetTemplate();
    template.setOptions(Collections.singletonMap("spark.hadoop.fs.azure.account.key.kylogreg1.blob.core.windows.net", azureAccountKey));
    final DataSource request = new DataSource();
    request.setConnector(connector);
    request.setTemplate(template);
    request.setTitle("test list files wasb");
    final DataSource dataSource = given(DataSourceController.BASE).when().body(request).post().then().statusCode(200).extract().as(DataSource.class);
    // Test listing containers
    final List<DataSetFile> containers = given(DataSourceController.BASE).when().pathParam("id", dataSource.getId()).queryParam("path", "wasb://kylogreg1.blob.core.windows.net/").get("{id}/files").then().statusCode(200).extract().as(DataSetFileList.class);
    Assert.assertThat(containers, CoreMatchers.hasItem(new CustomMatcher<DataSetFile>("DataSetFile name=blob123 directory=true") {

        @Override
        public boolean matches(final Object item) {
            return (item instanceof DataSetFile) && Objects.equals("blob123", ((DataSetFile) item).getName()) && Objects.equals("wasb://blob123@kylogreg1.blob.core.windows.net/", ((DataSetFile) item).getPath()) && ((DataSetFile) item).isDirectory();
        }
    }));
    // Test listing files
    final List<DataSetFile> files = given(DataSourceController.BASE).when().pathParam("id", dataSource.getId()).queryParam("path", "wasb://blob123@kylogreg1.blob.core.windows.net/").get("{id}/files").then().statusCode(200).extract().as(DataSetFileList.class);
    Assert.assertThat(files, CoreMatchers.hasItem(new CustomMatcher<DataSetFile>("DataSetFile name=books1.json directory=true") {

        @Override
        public boolean matches(final Object item) {
            return (item instanceof DataSetFile) && Objects.equals("books1.json", ((DataSetFile) item).getName()) && Objects.equals("wasb://blob123@kylogreg1.blob.core.windows.net/books1.json", ((DataSetFile) item).getPath()) && !((DataSetFile) item).isDirectory();
        }
    }));
}