Examples with DatasetSpecification co.cask.cdap.api.dataset.DatasetSpecification used on opensource projects

Example 61 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class DatasetDefinitionRegistryWithDefaultModules method testPFSReconfigure.

@Test
public void testPFSReconfigure() throws IncompatibleUpdateException {
    DatasetDefinition pfsDef = registry.get(PartitionedFileSet.class.getName());
    Assert.assertTrue(pfsDef instanceof Reconfigurable);
    DatasetProperties props = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addIntField("i").addStringField("s").build()).build();
    DatasetSpecification spec = pfsDef.configure("pfs", props);
    DatasetProperties noIprops = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").build()).build();
    try {
        ((Reconfigurable) pfsDef).reconfigure("pfs", noIprops, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    DatasetProperties longIprops = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addLongField("i").addStringField("s").build()).build();
    try {
        ((Reconfigurable) pfsDef).reconfigure("pfs", longIprops, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    DatasetProperties revProps = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").addIntField("i").build()).build();
    try {
        ((Reconfigurable) pfsDef).reconfigure("pfs", revProps, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    // test reconfiguring a PFS created before CDAP-13120, with no defaulted base path
    // no base path should be set for the reconfigured dataset either
    DatasetProperties oldProps = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").build()).add(PartitionedFileSetDefinition.NAME_AS_BASE_PATH_DEFAULT, "false").build();
    DatasetSpecification oldSpec = pfsDef.configure("pfs", oldProps);
    DatasetSpecification newSpec = ((Reconfigurable) pfsDef).reconfigure("pfs", oldProps, oldSpec);
    // make sure base path is not set
    Assert.assertNull(newSpec.getSpecification("files").getProperty(FileSetProperties.BASE_PATH));
    // test reconfiguring a PFS created after CDAP-13120, where base path is default to the dataset name
    props = PartitionedFileSetProperties.builder().setPartitioning(Partitioning.builder().addStringField("s").build()).build();
    oldSpec = pfsDef.configure("pfs", props);
    newSpec = ((Reconfigurable) pfsDef).reconfigure("pfs", props, oldSpec);
    // make sure base path is similarly set, even when not explicitly given
    Assert.assertEquals("pfs", newSpec.getSpecification("files").getProperty(FileSetProperties.BASE_PATH));
    // make sure it is set for subsequent reconfigures as well
    newSpec = ((Reconfigurable) pfsDef).reconfigure("pfs", props, oldSpec);
    // make sure base path is similarly set, even when not explicitly given
    Assert.assertEquals("pfs", newSpec.getSpecification("files").getProperty(FileSetProperties.BASE_PATH));
}

Example 62 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class DatasetDefinitionRegistryWithDefaultModules method testIndexedTableReconfigure.

@Test
public void testIndexedTableReconfigure() throws IncompatibleUpdateException {
    DatasetDefinition indexedTableDef = registry.get(IndexedTable.class.getName());
    Assert.assertTrue(indexedTableDef instanceof Reconfigurable);
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(false).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "a,b,c").build();
    DatasetSpecification spec = indexedTableDef.configure("idxtb", props);
    DatasetProperties compat = TableProperties.builder().setReadlessIncrementSupport(// turning on is ok
    true).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "c,b,a").build();
    spec = ((Reconfigurable) indexedTableDef).reconfigure("idxtb", compat, spec);
    DatasetProperties incompat = TableProperties.builder().setReadlessIncrementSupport(true).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "a,d").build();
    try {
        ((Reconfigurable) indexedTableDef).reconfigure("idxtb", incompat, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
    incompat = TableProperties.builder().setReadlessIncrementSupport(// turning off is not ok
    false).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "a,b,c").build();
    try {
        ((Reconfigurable) indexedTableDef).reconfigure("idxtb", incompat, spec);
        Assert.fail("reconfigure should have thrown exception");
    } catch (IncompatibleUpdateException e) {
    // expected
    }
}

Example 63 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class LevelDBStreamConsumerStateStoreFactory method getLevelDBTableAdmin.

private LevelDBTableAdmin getLevelDBTableAdmin(TableId tableId) throws IOException {
    DatasetProperties props = TableProperties.builder().setColumnFamily("t").build();
    LevelDBTableDefinition tableDefinition = new LevelDBTableDefinition("tableDefinition");
    DatasetSpecification spec = tableDefinition.configure(tableId.getTableName(), props);
    return new LevelDBTableAdmin(DatasetContext.from(tableId.getNamespace()), spec, tableService, cConf);
}

Example 64 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class HBaseDatasetMetricsReporter method report.

private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException {
    Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap();
    for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) {
        String hbaseNamespace = statEntry.getKey().getNamespace();
        String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace);
        // tableNames that doesn't start with user are ignored
        if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) {
            continue;
        }
        String tableName = statEntry.getKey().getTableName();
        try {
            Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace));
            for (DatasetSpecificationSummary spec : instances) {
                DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName()));
                if (specification.isParent(tableName)) {
                    MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName()));
                    collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB());
                    break;
                }
            }
        } catch (DatasetManagementException | ServiceUnavailableException e) {
        // No op
        }
    }
}

Example 65 with DatasetSpecification

use of co.cask.cdap.api.dataset.DatasetSpecification in project cdap by caskdata.

the class PartitionedFileSetDefinition method reconfigure.

@Override
public DatasetSpecification reconfigure(String instanceName, DatasetProperties properties, DatasetSpecification currentSpec) throws IncompatibleUpdateException {
    // validate that the partitioning is not changing
    Partitioning oldPartitioning = PartitionedFileSetProperties.getPartitioning(currentSpec.getProperties());
    Partitioning newPartitioning = PartitionedFileSetProperties.getPartitioning(properties.getProperties());
    Preconditions.checkNotNull(oldPartitioning, "Existing dataset has no partitioning");
    Preconditions.checkNotNull(newPartitioning, "New properties do not contain partitioning");
    if (!Iterators.elementsEqual(oldPartitioning.getFields().entrySet().iterator(), newPartitioning.getFields().entrySet().iterator())) {
        throw new IncompatibleUpdateException(String.format("Partitioning cannot be changed. Existing: %s, new: %s", oldPartitioning, newPartitioning));
    }
    Map<String, String> pfsProperties = new HashMap<>(properties.getProperties());
    // define the columns for indexing on the partitionsTable
    DatasetProperties indexedTableProperties = DatasetProperties.builder().addAll(properties.getProperties()).add(IndexedTable.INDEX_COLUMNS_CONF_KEY, INDEXED_COLS).build();
    // only set the default base path property if the default was set the last time it was configured,
    // and no base path is in the current properties.
    DatasetSpecification currentFileSpec = currentSpec.getSpecification(FILESET_NAME);
    DatasetProperties.Builder newFileProperties = DatasetProperties.builder().addAll(properties.getProperties());
    String useNameAsBasePathDefault = currentSpec.getProperty(NAME_AS_BASE_PATH_DEFAULT);
    if (Boolean.parseBoolean(useNameAsBasePathDefault) && !properties.getProperties().containsKey(FileSetProperties.BASE_PATH)) {
        newFileProperties.add(FileSetProperties.BASE_PATH, instanceName);
        pfsProperties.put(NAME_AS_BASE_PATH_DEFAULT, Boolean.TRUE.toString());
    }
    return DatasetSpecification.builder(instanceName, getName()).properties(pfsProperties).datasets(AbstractDatasetDefinition.reconfigure(filesetDef, FILESET_NAME, newFileProperties.build(), currentFileSpec), AbstractDatasetDefinition.reconfigure(indexedTableDef, PARTITION_TABLE_NAME, indexedTableProperties, currentSpec.getSpecification(PARTITION_TABLE_NAME))).build();
}