Examples with DatasetAdmin io.cdap.cdap.api.dataset.DatasetAdmin used on opensource projects

Example 66 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class BufferingTableTest method testChangingParamsAndReturnValues.

@Test
public void testChangingParamsAndReturnValues() throws Exception {
    // The test verifies that one can re-use byte arrays passed as parameters to write methods of a table without
    // affecting the stored data.
    // Also, one can re-use (modify) returned data from the table without affecting the stored data.
    DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(isReadlessIncrementSupported()).build();
    DatasetAdmin admin = getTableAdmin(CONTEXT1, MY_TABLE, props);
    admin.create();
    try (BufferingTable table = getTable(CONTEXT1, MY_TABLE, props)) {
        // writing some data: we'll need it to test delete later
        Transaction tx = txClient.startShort();
        table.startTx(tx);
        table.put(new byte[] { 0 }, new byte[] { 9 }, new byte[] { 8 });
        table.commitTx();
        txClient.commitOrThrow(tx);
        // start new for in-mem buffer behavior testing
        tx = txClient.startShort();
        table.startTx(tx);
        // write some data but not commit
        byte[] rowParam = new byte[] { 1 };
        byte[] colParam = new byte[] { 2 };
        byte[] valParam = Bytes.toBytes(3L);
        table.put(rowParam, colParam, valParam);
        verify123(table);
        // change passed earlier byte arrays in place, this should not affect stored previously values
        rowParam[0]++;
        colParam[0]++;
        valParam[0]++;
        verify123(table);
        // try get row and change returned values in place, which should not affect the data stored
        Row getRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getRowResult = getRow.getColumns();
        Assert.assertEquals(1, getRowResult.size());
        byte[] colFromGetRow = getRowResult.keySet().iterator().next();
        byte[] valFromGetRow = getRowResult.get(colFromGetRow);
        getRowResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetRow);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetRow);
        colFromGetRow[0]++;
        valFromGetRow[0]++;
        verify123(table);
        // try get set of columns in a row and change returned values in place, which should not affect the data stored
        Row getColumnSetRow = table.get(new byte[] { 1 });
        Map<byte[], byte[]> getColumnSetResult = getColumnSetRow.getColumns();
        Assert.assertEquals(1, getColumnSetResult.size());
        byte[] colFromGetColumnSet = getColumnSetResult.keySet().iterator().next();
        byte[] valFromGetColumnSet = getColumnSetResult.values().iterator().next();
        getColumnSetResult.remove(new byte[] { 2 });
        Assert.assertArrayEquals(new byte[] { 2 }, colFromGetColumnSet);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumnSet);
        colFromGetColumnSet[0]++;
        valFromGetColumnSet[0]++;
        verify123(table);
        // try get column and change returned value in place, which should not affect the data stored
        byte[] valFromGetColumn = table.get(new byte[] { 1 }, new byte[] { 2 });
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromGetColumn);
        valFromGetColumn[0]++;
        verify123(table);
        // try scan and change returned value in place, which should not affect the data stored
        Scanner scan = table.scan(new byte[] { 1 }, null);
        Row next = scan.next();
        Assert.assertNotNull(next);
        byte[] rowFromScan = next.getRow();
        Assert.assertArrayEquals(new byte[] { 1 }, rowFromScan);
        Map<byte[], byte[]> cols = next.getColumns();
        Assert.assertEquals(1, cols.size());
        byte[] colFromScan = cols.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromScan);
        byte[] valFromScan = next.get(new byte[] { 2 });
        Assert.assertNotNull(valFromScan);
        Assert.assertArrayEquals(Bytes.toBytes(3L), valFromScan);
        Assert.assertNull(scan.next());
        cols.remove(new byte[] { 2 });
        rowFromScan[0]++;
        colFromScan[0]++;
        valFromScan[0]++;
        verify123(table);
        // try delete and change params in place: this should not affect stored data
        rowParam = new byte[] { 1 };
        colParam = new byte[] { 2 };
        table.delete(rowParam, colParam);
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        rowParam[0] = 0;
        colParam[0] = 9;
        Assert.assertNull(table.get(new byte[] { 1 }, new byte[] { 2 }));
        Assert.assertArrayEquals(new byte[] { 8 }, table.get(new byte[] { 0 }, new byte[] { 9 }));
        // try increment column and change params in place: this should not affect stored data
        byte[] rowIncParam = new byte[] { 1 };
        byte[] colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, 3);
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment set of columns and change params in place, try also to change values in returned map: this all
        // should not affect stored data.
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try increment and change returned values: should not affect the stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        Row countersRow = table.incrementAndGet(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        Map<byte[], byte[]> counters = countersRow.getColumns();
        Assert.assertEquals(1, counters.size());
        byte[] colFromInc = counters.keySet().iterator().next();
        Assert.assertArrayEquals(new byte[] { 2 }, colFromInc);
        Assert.assertEquals(3, Bytes.toLong(counters.get(colFromInc)));
        counters.remove(new byte[] { 2 });
        colFromInc[0]++;
        verify123(table);
        // try increment write and change params in place: this should not affect stored data
        rowIncParam = new byte[] { 1 };
        colIncParam = new byte[] { 2 };
        table.increment(rowIncParam, colIncParam, -1);
        table.increment(rowIncParam, new byte[][] { colIncParam }, new long[] { 1 });
        verify123(table);
        rowIncParam[0]++;
        colIncParam[0]++;
        verify123(table);
        // try compareAndSwap and change params in place: this should not affect stored data
        byte[] rowSwapParam = new byte[] { 1 };
        byte[] colSwapParam = new byte[] { 2 };
        byte[] valSwapParam = Bytes.toBytes(3L);
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(3L), Bytes.toBytes(4L));
        table.compareAndSwap(rowSwapParam, colSwapParam, Bytes.toBytes(4L), valSwapParam);
        verify123(table);
        rowSwapParam[0]++;
        colSwapParam[0]++;
        valSwapParam[0]++;
        verify123(table);
    // We don't care to persist changes and commit tx here: we tested what we wanted
    } finally {
        admin.drop();
    }
}

Example 67 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class DefaultMetricDatasetFactory method getOrCreateTable.

private MetricsTable getOrCreateTable(DatasetId tableId, DatasetProperties props) throws IOException {
    DatasetContext datasetContext = DatasetContext.from(NamespaceId.SYSTEM.getNamespace());
    DatasetSpecification spec = metricsTableDefinition.configure(tableId.getDataset(), props);
    if (!existingDatasets.contains(tableId)) {
        // Check and create if we don't know if the table exists or not
        DatasetAdmin admin = metricsTableDefinition.getAdmin(datasetContext, spec, getClass().getClassLoader());
        if (!admin.exists()) {
            // All admin.create() implementations handled race condition for concurrent create.
            // Not sure if that's the API contract or just the implementations since it is not specified in the API
            // But from the dataset op executor implementation, it seems it is a required contract.
            admin.create();
        }
        existingDatasets.add(tableId);
    }
    return metricsTableDefinition.getDataset(datasetContext, spec, Collections.emptyMap(), getClass().getClassLoader());
}

Example 68 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class SystemDatasetRuntimeModule method getDistributedModules.

@Override
public Module getDistributedModules() {
    return new AbstractModule() {

        @Override
        protected void configure() {
            MapBinder<String, DatasetModule> mapBinder = MapBinder.newMapBinder(binder(), String.class, DatasetModule.class, Constants.Dataset.Manager.DefaultDatasetModules.class);
            // NOTE: order is important due to dependencies between modules
            mapBinder.addBinding("orderedTable-hbase").toProvider(OrderedTableModuleProvider.class).in(Singleton.class);
            mapBinder.addBinding("metricsTable-hbase").toInstance(new HBaseMetricsTableModule());
            bindDefaultModules(mapBinder);
            bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).toInstance("table");
            bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).to(HBaseTableDefinition.class);
            bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).toInstance("table-no-tx");
            bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).to(HBaseMetricsTableDefinition.class);
            // Direct binding for the Metrics table definition such that metrics system doesn't need to go through
            // dataset service to get metrics table.
            bind(new TypeLiteral<DatasetDefinition<MetricsTable, DatasetAdmin>>() {
            }).toInstance(new HBaseMetricsTableDefinition(MetricsTable.class.getName()));
        }
    };
}

Example 69 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class SystemDatasetRuntimeModule method getInMemoryModules.

@Override
public Module getInMemoryModules() {
    return new AbstractModule() {

        @Override
        protected void configure() {
            MapBinder<String, DatasetModule> mapBinder = MapBinder.newMapBinder(binder(), String.class, DatasetModule.class, Constants.Dataset.Manager.DefaultDatasetModules.class);
            // NOTE: order is important due to dependencies between modules
            mapBinder.addBinding("orderedTable-memory").toInstance(new InMemoryTableModule());
            mapBinder.addBinding("metricsTable-memory").toInstance(new InMemoryMetricsTableModule());
            bindDefaultModules(mapBinder);
            bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).toInstance("table");
            bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).to(InMemoryTableDefinition.class);
            bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).toInstance("table-no-tx");
            bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).to(InMemoryMetricsTableDefinition.class);
            // Direct binding for the Metrics table definition such that metrics system doesn't need to go through
            // dataset service to get metrics table.
            bind(new TypeLiteral<DatasetDefinition<MetricsTable, DatasetAdmin>>() {
            }).toInstance(new InMemoryMetricsTableDefinition(MetricsTable.class.getName()));
        }
    };
}

Example 70 with DatasetAdmin

use of io.cdap.cdap.api.dataset.DatasetAdmin in project cdap by caskdata.

the class DatasetAdminService method drop.

public void drop(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetSpecification spec) throws Exception {
    LOG.info("Dropping dataset with spec: {}, type meta: {}", spec, typeMeta);
    try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
        UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
        ImpersonationUtils.doAs(ugi, (Callable<Void>) () -> {
            DatasetType type = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
            if (type == null) {
                throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
            }
            DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec);
            try {
                admin.drop();
            } finally {
                Closeables.closeQuietly(admin);
            }
            return null;
        });
    }
}