Examples with Datasource com.thinkbiganalytics.metadata.rest.model.data.Datasource used on opensource projects

Example 6 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class MetadataClientProviderTest method testEnsureHiveTableDatasource.

@Test
public void testEnsureHiveTableDatasource() {
    this.provider.ensureHiveTableDatasource("test7", "", "testdb", "test_table");
    Datasource ds = this.provider.getDatasourceByName("test7");
    assertThat(ds).isNotNull();
    assertThat(ds).isInstanceOf(HiveTableDatasource.class);
    String dsId = ds.getId();
    HiveTableDatasource dds = (HiveTableDatasource) ds;
    assertThat(dds.getTableName()).contains("test_table");
    ds = this.provider.ensureHiveTableDatasource("test7", "", "testdb", "test_table");
    assertThat(ds).isNotNull();
    assertThat(ds.getId()).isEqualTo(dsId);
}

Example 7 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class MetadataClientProviderTest method testBeginOperation.

@Test
public void testBeginOperation() {
    Feed feed = this.provider.ensureFeed("category", "test9", "");
    Datasource ds = this.provider.ensureDirectoryDatasource("test9", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedDestination(feed.getId(), ds.getId());
    FeedDestination dest = feed.getDestination(ds.getId());
    DataOperation op = this.provider.beginOperation(dest, new DateTime());
    assertThat(op).isNotNull();
    assertThat(op.getState()).isEqualTo(State.IN_PROGRESS);
}

Example 8 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class FeedImporter method validateUserDatasources.

/**
 * Validates that user data sources can be imported with provided properties.
 *
 * @return {@code true} if the feed can be imported, or {@code false} otherwise
 */
private boolean validateUserDatasources() {
    FeedMetadata metadata = importFeed.getFeedToImport();
    final UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importFeed.getImportOptions().getUploadKey(), "Validating data sources.");
    // Get data sources needing to be created
    final Set<String> availableDatasources = metadataAccess.read(() -> datasourceProvider.getDatasources(datasourceProvider.datasetCriteria().type(UserDatasource.class)).stream().map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId).map(Object::toString).collect(Collectors.toSet()));
    final ImportComponentOption componentOption = importFeedOptions.findImportComponentOption(ImportComponent.USER_DATASOURCES);
    final List<Datasource> providedDatasources = Optional.ofNullable(metadata.getUserDatasources()).orElse(Collections.emptyList());
    if (componentOption.getProperties().isEmpty()) {
        componentOption.setProperties(providedDatasources.stream().filter(datasource -> !availableDatasources.contains(datasource.getId())).map(datasource -> new ImportProperty(datasource.getName(), datasource.getId(), null, null, null)).collect(Collectors.toList()));
    }
    // Update feed with re-mapped data sources
    final boolean valid = componentOption.getProperties().stream().allMatch(property -> {
        if (property.getPropertyValue() != null) {
            ImportUtil.replaceDatasource(metadata, property.getProcessorId(), property.getPropertyValue());
            return true;
        } else {
            return false;
        }
    });
    if (valid) {
        statusMessage.update("Validated data sources.", true);
    } else {
        statusMessage.update("Validation Error. Additional properties are needed before uploading the feed.", false);
        importFeed.setValid(false);
    }
    uploadProgressService.completeSection(importFeed.getImportOptions(), ImportSection.Section.VALIDATE_USER_DATASOURCES);
    return valid;
}

Example 9 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class DatasourceController method describeTable.

/**
 * Gets the schema of the specified table using the specified data source.
 *
 * @param idStr     the data source id
 * @param tableName the table name
 * @param schema    the schema name, or {@code null} to search all schemas
 * @return the table and field details
 */
@GET
@Path("{id}/tables/{tableName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the schema of the specified table.", notes = "Connects to the database specified by the data source.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the table schema.", response = TableSchema.class), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response describeTable(@PathParam("id") final String idStr, @PathParam("tableName") final String tableName, @QueryParam("schema") final String schema) {
    // Verify user has access to data source
    final Optional<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> id = metadata.read(() -> {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasetProvider.getDatasource(datasetProvider.resolve(idStr));
        return Optional.ofNullable(datasource).map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId);
    });
    // Retrieve table description using system user
    return metadata.read(() -> {
        final TableSchema tableSchema = id.map(datasetProvider::getDatasource).map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(JdbcDatasource.class::isInstance).map(JdbcDatasource.class::cast).map(datasource -> dbcpConnectionPoolTableInfo.describeTableForDatasource(datasource, schema, tableName)).orElseThrow(() -> new NotFoundException("No JDBC datasource exists with the given ID: " + idStr));
        return Response.ok(tableSchema).build();
    }, MetadataAccess.SERVICE);
}

Example 10 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class DatasourceController method getDatasource.

/**
 * Gets the datasource with the id provided.
 *
 * @param idStr     the datasource id
 * @param sensitive {@code true} to include sensitive fields in the response, or {@code false} otherwise
 * @return the datasource object
 * @throws AccessControlException if the user does not have the {@code ACCESS_DATASOURCES} permission
 */
@GET
@Path("{id}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the data source with the provided id.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the data source.", response = Datasource.class), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "The data source does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "Kylo is unavailable.", response = RestResponseStatus.class) })
public Datasource getDatasource(@PathParam("id") String idStr, @QueryParam("sensitive") boolean sensitive) {
    return this.metadata.read(() -> {
        // Check permissions
        accessController.checkPermission(AccessController.SERVICES, sensitive ? FeedServicesAccessControl.ADMIN_DATASOURCES : FeedServicesAccessControl.ACCESS_DATASOURCES);
        com.thinkbiganalytics.metadata.api.datasource.Datasource.ID id = this.datasetProvider.resolve(idStr);
        com.thinkbiganalytics.metadata.api.datasource.Datasource ds = this.datasetProvider.getDatasource(id);
        if (ds != null) {
            final Datasource restModel = datasourceTransform.toDatasource(ds, sensitive ? DatasourceModelTransform.Level.ADMIN : DatasourceModelTransform.Level.FULL);
            if (ds instanceof AccessControlled) {
                securityTransform.applyAccessControl((AccessControlled) ds, restModel);
            }
            return restModel;
        } else {
            throw new NotFoundException("No datasource exists with the given ID: " + idStr);
        }
    });
}