Examples with DatasetSpecification co.cask.cdap.api.dataset.DatasetSpecification used on opensource projects

Example 66 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class CubeDatasetDefinition method reconfigure.

@Override
public DatasetSpecification reconfigure(String instanceName, DatasetProperties newProps, DatasetSpecification currentSpec) throws IncompatibleUpdateException {
    DatasetProperties factTableProperties = computeFactTableProperties(newProps);
    List<DatasetSpecification> datasetSpecs = Lists.newArrayList();
    // Configuring table that hold mappings of tag names and values and such
    datasetSpecs.add(reconfigure(metricsTableDef, "entity", newProps, currentSpec.getSpecification("entity")));
    for (int resolution : getResolutions(newProps.getProperties())) {
        String factTableName = String.valueOf(resolution);
        DatasetSpecification existing = currentSpec.getSpecification(factTableName);
        DatasetSpecification factTableSpec = existing == null ? tableDef.configure(factTableName, factTableProperties) : reconfigure(tableDef, factTableName, factTableProperties, existing);
        datasetSpecs.add(factTableSpec);
    }
    return DatasetSpecification.builder(instanceName, getName()).properties(newProps.getProperties()).datasets(datasetSpecs).build();
}

Example 67 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class ObjectMappedTableDefinition method getDataset.

@Override
public ObjectMappedTableDataset<?> getDataset(DatasetContext datasetContext, DatasetSpecification spec, Map<String, String> arguments, ClassLoader classLoader) throws IOException {
    String keyName = ObjectMappedTableProperties.getRowKeyExploreName(spec.getProperties());
    DatasetSpecification tableSpec = spec.getSpecification(TABLE_NAME);
    // TODO: remove after CDAP-2122 is done
    if (!tableSpec.getProperties().containsKey(Table.PROPERTY_SCHEMA)) {
        tableSpec = DatasetSpecification.builder(tableSpec.getName(), tableSpec.getType()).properties(tableSpec.getProperties()).property(Table.PROPERTY_SCHEMA, spec.getProperty(Table.PROPERTY_SCHEMA)).property(Table.PROPERTY_SCHEMA_ROW_FIELD, keyName).datasets(tableSpec.getSpecifications().values()).build();
    }
    // reconstruct the table schema here because of backwards compatibility
    DatasetDefinition<Table, DatasetAdmin> tableDef = getDelegate(TABLE_NAME);
    Table table = tableDef.getDataset(datasetContext, tableSpec, arguments, classLoader);
    Map<String, String> properties = spec.getProperties();
    TypeRepresentation typeRep = GSON.fromJson(ObjectMappedTableProperties.getObjectTypeRepresentation(properties), TypeRepresentation.class);
    Schema objSchema = ObjectMappedTableProperties.getObjectSchema(properties);
    return new ObjectMappedTableDataset(spec.getName(), table, typeRep, objSchema, classLoader);
}

Example 68 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class MetricHBaseTableUtilTest method testGetVersion.

@Test
public void testGetVersion() throws Exception {
    // Verify new metric datasets are properly recognized as 2.8+ version from now on
    HBaseMetricsTableDefinition definition = new HBaseMetricsTableDefinition("foo", TEST_HBASE.getConfiguration(), hBaseTableUtil, new LocalLocationFactory(TMP_FOLDER.newFolder()), cConf);
    DatasetSpecification spec = definition.configure("metricV2.8", DatasetProperties.EMPTY);
    DatasetAdmin admin = definition.getAdmin(DatasetContext.from(NamespaceId.SYSTEM.getNamespace()), spec, null);
    admin.create();
    MetricHBaseTableUtil util = new MetricHBaseTableUtil(hBaseTableUtil);
    HBaseAdmin hAdmin = TEST_HBASE.getHBaseAdmin();
    TableId hTableId = hBaseTableUtil.createHTableId(NamespaceId.SYSTEM, spec.getName());
    HTableDescriptor desc = hBaseTableUtil.getHTableDescriptor(hAdmin, hTableId);
    Assert.assertEquals(MetricHBaseTableUtil.Version.VERSION_2_8_OR_HIGHER, util.getVersion(desc));
    // Verify HBase table without coprocessor is properly recognized as 2.6- version
    TableName table26 = TableName.valueOf("metricV2.6");
    hAdmin.createTable(new HTableDescriptor(table26));
    desc = hAdmin.getTableDescriptor(table26);
    Assert.assertEquals(MetricHBaseTableUtil.Version.VERSION_2_6_OR_LOWER, util.getVersion(desc));
    // Verify HBase table with IncrementHandler coprocessor but without cdap.version on it is properly recognized as
    // 2.7 version
    TableName table27 = TableName.valueOf("metricV2.7");
    desc = new HTableDescriptor(table27);
    desc.addCoprocessor(hBaseTableUtil.getIncrementHandlerClassForVersion().getName());
    hAdmin.createTable(desc);
    desc = hAdmin.getTableDescriptor(table27);
    Assert.assertEquals(MetricHBaseTableUtil.Version.VERSION_2_7, util.getVersion(desc));
}

Example 69 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class ApplicationVerificationStage method verifyData.

private void verifyData(ApplicationId appId, ApplicationSpecification specification, @Nullable KerberosPrincipalId ownerPrincipal) throws Exception {
    // NOTE: no special restrictions on dataset module names, etc
    VerifyResult result;
    for (DatasetCreationSpec dataSetCreateSpec : specification.getDatasets().values()) {
        result = getVerifier(DatasetCreationSpec.class).verify(appId, dataSetCreateSpec);
        if (!result.isSuccess()) {
            throw new RuntimeException(result.getMessage());
        }
        String dsName = dataSetCreateSpec.getInstanceName();
        final DatasetId datasetInstanceId = appId.getParent().dataset(dsName);
        // get the authorizing user
        String authorizingUser = AuthorizationUtil.getAppAuthorizingUser(ownerAdmin, authenticationContext, appId, ownerPrincipal);
        DatasetSpecification existingSpec = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<DatasetSpecification>() {

            @Override
            public DatasetSpecification call() throws Exception {
                return dsFramework.getDatasetSpec(datasetInstanceId);
            }
        });
        if (existingSpec != null && !existingSpec.getType().equals(dataSetCreateSpec.getTypeName())) {
            // New app trying to deploy an dataset with same instanceName but different Type than that of existing.
            throw new DataSetException(String.format("Cannot Deploy Dataset : %s with Type : %s : Dataset with different Type Already Exists", dsName, dataSetCreateSpec.getTypeName()));
        }
        // if the dataset existed verify its owner is same.
        if (existingSpec != null) {
            verifyOwner(datasetInstanceId, ownerPrincipal);
        }
    }
    for (StreamSpecification spec : specification.getStreams().values()) {
        result = getVerifier(StreamSpecification.class).verify(appId, spec);
        if (!result.isSuccess()) {
            throw new RuntimeException(result.getMessage());
        }
        // if the stream existed verify the owner to be the same
        if (store.getStream(appId.getNamespaceId(), spec.getName()) != null) {
            verifyOwner(appId.getParent().stream(spec.getName()), ownerPrincipal);
        }
    }
}

Example 70 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class DatasetInstanceCreator method createInstances.

/**
 * Receives an input containing application specification and location
 * and verifies both.
 *
 * @param namespaceId the namespace to create the dataset instance in
 * @param datasets the datasets to create
 * @param ownerPrincipal the principal of the owner for the datasets to be created.
 * @param authorizingUser the authorizing user who will be making the call
 */
void createInstances(NamespaceId namespaceId, Map<String, DatasetCreationSpec> datasets, @Nullable final KerberosPrincipalId ownerPrincipal, String authorizingUser) throws Exception {
    // create dataset instances
    for (Map.Entry<String, DatasetCreationSpec> instanceEntry : datasets.entrySet()) {
        String instanceName = instanceEntry.getKey();
        final DatasetId instanceId = namespaceId.dataset(instanceName);
        final DatasetCreationSpec instanceSpec = instanceEntry.getValue();
        DatasetSpecification existingSpec = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<DatasetSpecification>() {

            @Override
            public DatasetSpecification call() throws Exception {
                return datasetFramework.getDatasetSpec(instanceId);
            }
        });
        if (existingSpec == null) {
            LOG.info("Adding dataset instance: {}", instanceName);
            AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, instanceSpec.getProperties(), ownerPrincipal);
                    return null;
                }
            });
        } else {
            if (!existingSpec.getType().equals(instanceSpec.getTypeName())) {
                throw new IncompatibleUpdateException(String.format("Existing dataset '%s' of type '%s' may not be updated to type '%s'", instanceName, existingSpec.getType(), instanceSpec.getTypeName()));
            }
            if (allowDatasetUncheckedUpgrade) {
                LOG.info("Updating dataset instance: {}", instanceName);
                AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Void>() {

                    @Override
                    public Void call() throws Exception {
                        datasetFramework.updateInstance(instanceId, instanceSpec.getProperties());
                        return null;
                    }
                });
            }
        }
    }
}