Examples with Update io.cdap.cdap.data2.dataset2.lib.table.Update used on opensource projects

Example 6 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class InMemoryTableService method swap.

public static synchronized boolean swap(String tableName, byte[] row, byte[] column, byte[] oldValue, byte[] newValue) {
    ConcurrentNavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, Update>>> table = tables.get(tableName);
    // get the correct row from the table, create it if it doesn't exist
    NavigableMap<byte[], NavigableMap<Long, Update>> rowMap = table.get(row);
    Update existingValue = null;
    if (rowMap != null) {
        NavigableMap<Long, Update> columnMap = rowMap.get(column);
        if (columnMap != null) {
            existingValue = columnMap.lastEntry().getValue();
        }
    }
    // verify existing value matches
    if (oldValue == null && existingValue != null) {
        return false;
    }
    if (oldValue != null && (existingValue == null || !Bytes.equals(oldValue, existingValue.getBytes()))) {
        return false;
    }
    // write new value
    if (newValue == null) {
        if (rowMap != null) {
            rowMap.remove(column);
        }
    } else {
        if (rowMap == null) {
            rowMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
            table.put(row, rowMap);
        }
        NavigableMap<Long, Update> columnMap = rowMap.get(column);
        if (columnMap == null) {
            columnMap = Maps.newTreeMap();
            rowMap.put(column, columnMap);
        }
        PutValue newPut = new PutValue(newValue);
        columnMap.put(System.currentTimeMillis(), newPut);
    }
    return true;
}

Example 7 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testBasicManagement.

@Test
public void testBasicManagement() throws Exception {
    DatasetTypeId tableType = NAMESPACE_ID.datasetType(Table.class.getName());
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(FILE, new FileSetModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // keyvalue has been added in the system namespace
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    DatasetSpecification spec = framework.getDatasetSpec(MY_TABLE);
    Assert.assertNotNull(spec);
    Assert.assertEquals(MY_TABLE.getEntityName(), spec.getName());
    Assert.assertEquals(Table.class.getName(), spec.getType());
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE2));
    // Update instances
    File baseDir = TMP_FOLDER.newFolder();
    framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    // this should fail because it would "internalize" external data
    try {
        framework.updateInstance(MY_DS, DatasetProperties.EMPTY);
        Assert.fail("update should have thrown instance conflict");
    } catch (InstanceConflictException e) {
    // expected
    }
    baseDir = TMP_FOLDER.newFolder();
    // this should succeed because it simply changes the external path
    framework.updateInstance(MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    spec = framework.getDatasetSpec(MY_DS);
    Assert.assertNotNull(spec);
    Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
    // cleanup
    try {
        framework.deleteAllModules(NAMESPACE_ID);
        Assert.fail("should not delete modules: there are datasets using their types");
    } catch (DatasetManagementException e) {
    // expected
    }
    // types are still there
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    framework.deleteAllInstances(NAMESPACE_ID);
    Assert.assertEquals(0, framework.getInstances(NAMESPACE_ID).size());
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE));
    Assert.assertFalse(framework.hasInstance(MY_TABLE2));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE2));
    // now it should succeed
    framework.deleteAllModules(NAMESPACE_ID);
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasType(tableType));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
}

Example 8 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testAuditPublish.

@Test
public void testAuditPublish() throws Exception {
    // Clear all audit messages
    inMemoryAuditPublisher.popMessages();
    List<AuditMessage> expectedMessages = new ArrayList<>();
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE2, "", AuditType.CREATE, AuditPayload.EMPTY_PAYLOAD));
    // Update instance
    framework.updateInstance(MY_TABLE, DatasetProperties.EMPTY);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.UPDATE, AuditPayload.EMPTY_PAYLOAD));
    // Access instance
    ProgramRunId runId = new ProgramId("ns", "app", ProgramType.FLOW, "flow").run(RunIds.generate().getId());
    LineageWriterDatasetFramework lineageFramework = new LineageWriterDatasetFramework(framework, new NoOpLineageWriter(), new NoOpUsageRegistry(), new AuthenticationTestContext(), new NoOpAuthorizer());
    lineageFramework.setContext(new TestProgramContext(runId));
    lineageFramework.setAuditPublisher(inMemoryAuditPublisher);
    lineageFramework.getDataset(MY_TABLE, ImmutableMap.<String, String>of(), getClass().getClassLoader());
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.ACCESS, new AccessPayload(AccessType.UNKNOWN, runId)));
    // Truncate instance
    framework.truncateInstance(MY_TABLE);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.TRUNCATE, AuditPayload.EMPTY_PAYLOAD));
    // Delete instance
    framework.deleteInstance(MY_TABLE);
    expectedMessages.add(new AuditMessage(0, MY_TABLE, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    // Delete all instances in a namespace
    framework.deleteAllInstances(MY_TABLE2.getParent());
    expectedMessages.add(new AuditMessage(0, MY_TABLE2, "", AuditType.DELETE, AuditPayload.EMPTY_PAYLOAD));
    Assert.assertEquals(expectedMessages, inMemoryAuditPublisher.popMessages());
    // cleanup
    framework.deleteModule(IN_MEMORY);
}

Example 9 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class HBaseTable method persist.

@Override
protected void persist(NavigableMap<byte[], NavigableMap<byte[], Update>> updates) throws Exception {
    if (updates.isEmpty()) {
        return;
    }
    byte[] txId = tx == null ? null : Bytes.toBytes(tx.getTransactionId());
    byte[] txWritePointer = tx == null ? null : Bytes.toBytes(tx.getWritePointer());
    List<Mutation> mutations = new ArrayList<>();
    List<Increment> increments = new ArrayList<>();
    for (Map.Entry<byte[], NavigableMap<byte[], Update>> row : updates.entrySet()) {
        // create these only when they are needed
        PutBuilder put = null;
        PutBuilder incrementPut = null;
        IncrementBuilder increment = null;
        for (Map.Entry<byte[], Update> column : row.getValue().entrySet()) {
            // we want support tx and non-tx modes
            if (tx != null) {
                // TODO: hijacking timestamp... bad
                Update val = column.getValue();
                if (val instanceof IncrementValue) {
                    if (safeReadlessIncrements) {
                        increment = getIncrement(increment, row.getKey(), txId, txWritePointer);
                        increment.add(columnFamily, column.getKey(), tx.getWritePointer(), ((IncrementValue) val).getValue());
                    } else {
                        incrementPut = getPutForIncrement(incrementPut, row.getKey(), txId);
                        incrementPut.add(columnFamily, column.getKey(), tx.getWritePointer(), Bytes.toBytes(((IncrementValue) val).getValue()));
                    }
                } else if (val instanceof PutValue) {
                    put = getPut(put, row.getKey(), txId);
                    put.add(columnFamily, column.getKey(), tx.getWritePointer(), wrapDeleteIfNeeded(((PutValue) val).getValue()));
                }
            } else {
                Update val = column.getValue();
                if (val instanceof IncrementValue) {
                    incrementPut = getPutForIncrement(incrementPut, row.getKey(), txId);
                    incrementPut.add(columnFamily, column.getKey(), Bytes.toBytes(((IncrementValue) val).getValue()));
                } else if (val instanceof PutValue) {
                    put = getPut(put, row.getKey(), txId);
                    put.add(columnFamily, column.getKey(), ((PutValue) val).getValue());
                }
            }
        }
        if (incrementPut != null) {
            mutations.add(incrementPut.build());
        }
        if (increment != null) {
            increments.add(increment.build());
        }
        if (put != null) {
            mutations.add(put.build());
        }
    }
    if (!hbaseFlush(mutations) && increments.isEmpty()) {
        LOG.info("No writes to persist!");
    }
    if (!increments.isEmpty()) {
        table.batch(increments, new Object[increments.size()]);
    }
}

Example 10 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class HBaseTable method undo.

@Override
protected void undo(NavigableMap<byte[], NavigableMap<byte[], Update>> persisted) throws Exception {
    if (persisted.isEmpty()) {
        return;
    }
    // NOTE: we use Delete with the write pointer as the specific version to delete.
    List<Delete> deletes = Lists.newArrayList();
    for (Map.Entry<byte[], NavigableMap<byte[], Update>> row : persisted.entrySet()) {
        DeleteBuilder delete = tableUtil.buildDelete(row.getKey());
        delete.setAttribute(TX_MAX_LIFETIME_MILLIS_KEY, txMaxLifetimeMillis);
        for (Map.Entry<byte[], Update> column : row.getValue().entrySet()) {
            // we want support tx and non-tx modes
            if (tx != null) {
                delete.setAttribute(TxConstants.TX_ROLLBACK_ATTRIBUTE_KEY, new byte[0]);
                // TODO: hijacking timestamp... bad
                delete.deleteColumn(columnFamily, column.getKey(), tx.getWritePointer());
            } else {
                delete.deleteColumns(columnFamily, column.getKey());
            }
        }
        deletes.add(delete.build());
    }
    if (!deletes.isEmpty()) {
        hbaseDelete(deletes);
    }
}