Examples with Update io.cdap.cdap.data2.dataset2.lib.table.Update used on opensource projects

Example 31 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class InMemoryTableService method increment.

// todo: remove it from here: only used by "system" metrics table, which should be revised
@Deprecated
public static synchronized Map<byte[], Long> increment(String tableName, byte[] row, Map<byte[], Long> increments) {
    Map<byte[], Long> resultMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    ConcurrentNavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, Update>>> table = tables.get(tableName);
    // get the correct row from the table, create it if it doesn't exist
    NavigableMap<byte[], NavigableMap<Long, Update>> rowMap = table.get(row);
    if (rowMap == null) {
        rowMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
        table.put(row, rowMap);
    }
    // now increment each column, one by one
    long versionForWrite = System.currentTimeMillis();
    for (Map.Entry<byte[], Long> inc : increments.entrySet()) {
        IncrementValue increment = new IncrementValue(inc.getValue());
        // create the column in the row if it does not exist
        NavigableMap<Long, Update> colMap = rowMap.get(inc.getKey());
        Update last = null;
        if (colMap == null) {
            colMap = Maps.newTreeMap();
            rowMap.put(inc.getKey(), colMap);
        } else {
            last = colMap.lastEntry().getValue();
        }
        Update merged = Updates.mergeUpdates(last, increment);
        // put into the column with given version
        long newValue = Bytes.toLong(merged.getBytes());
        resultMap.put(inc.getKey(), newValue);
        colMap.put(versionForWrite, merged);
    }
    return resultMap;
}

Example 32 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class AbstractDatasetFrameworkTest method testBasicManagement.

@Test
public void testBasicManagement() throws Exception {
    DatasetTypeId tableType = NAMESPACE_ID.datasetType(Table.class.getName());
    // Adding modules
    DatasetFramework framework = getFramework();
    framework.addModule(IN_MEMORY, new InMemoryTableModule());
    framework.addModule(CORE, new CoreDatasetsModule());
    framework.addModule(FILE, new FileSetModule());
    framework.addModule(KEY_VALUE, new SingleTypeModule(SimpleKVTable.class));
    // keyvalue has been added in the system namespace
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasSystemType(SimpleKVTable.class.getName()));
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    // Creating instances
    framework.addInstance(Table.class.getName(), MY_TABLE, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE));
    DatasetSpecification spec = framework.getDatasetSpec(MY_TABLE);
    Assert.assertNotNull(spec);
    Assert.assertEquals(MY_TABLE.getEntityName(), spec.getName());
    Assert.assertEquals(Table.class.getName(), spec.getType());
    framework.addInstance(Table.class.getName(), MY_TABLE2, DatasetProperties.EMPTY);
    Assert.assertTrue(framework.hasInstance(MY_TABLE2));
    // Update instances
    File baseDir = TMP_FOLDER.newFolder();
    framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    // this should fail because it would "internalize" external data
    try {
        framework.updateInstance(MY_DS, DatasetProperties.EMPTY);
        Assert.fail("update should have thrown instance conflict");
    } catch (InstanceConflictException e) {
    // expected
    }
    baseDir = TMP_FOLDER.newFolder();
    // this should succeed because it simply changes the external path
    framework.updateInstance(MY_DS, FileSetProperties.builder().setBasePath(baseDir.getPath()).setDataExternal(true).build());
    spec = framework.getDatasetSpec(MY_DS);
    Assert.assertNotNull(spec);
    Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
    // cleanup
    try {
        framework.deleteAllModules(NAMESPACE_ID);
        Assert.fail("should not delete modules: there are datasets using their types");
    } catch (DatasetManagementException e) {
    // expected
    }
    // types are still there
    Assert.assertTrue(framework.hasType(tableType));
    Assert.assertTrue(framework.hasType(SIMPLE_KV_TYPE));
    framework.deleteAllInstances(NAMESPACE_ID);
    Assert.assertEquals(0, framework.getInstances(NAMESPACE_ID).size());
    Assert.assertFalse(framework.hasInstance(MY_TABLE));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE));
    Assert.assertFalse(framework.hasInstance(MY_TABLE2));
    Assert.assertNull(framework.getDatasetSpec(MY_TABLE2));
    // now it should succeed
    framework.deleteAllModules(NAMESPACE_ID);
    Assert.assertTrue(framework.hasSystemType(Table.class.getName()));
    Assert.assertFalse(framework.hasType(tableType));
    Assert.assertFalse(framework.hasType(SIMPLE_KV_TYPE));
}

Example 33 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class NoSqlStructuredTable method updateFieldsToBytes.

/**
 * Updates fields in the row and converts to a {@link Put} to write to table. The primary key must be provided.
 *
 * @param fields the fields to update
 * @return a PUT object
 * @throws InvalidFieldException if primary keys are missing or the column is not in schema
 */
private Put updateFieldsToBytes(Collection<Field<?>> fields) throws InvalidFieldException {
    Set<String> fieldNames = fields.stream().map(Field::getName).collect(Collectors.toSet());
    if (!fieldNames.containsAll(schema.getPrimaryKeys())) {
        throw new InvalidFieldException(schema.getTableId(), fields, String.format("Given fields %s does not contain all the " + "primary keys %s", fieldNames, schema.getPrimaryKeys()));
    }
    Set<String> primaryKeys = new HashSet<>(schema.getPrimaryKeys());
    Collection<Field<?>> keyFields = fields.stream().filter(field -> primaryKeys.contains(field.getName())).collect(Collectors.toList());
    byte[] primaryKey = convertKeyToBytes(keyFields, false);
    Put put = new Put(primaryKey);
    Row row = table.get(primaryKey);
    Map<String, Field<?>> fieldMap = fields.stream().collect(Collectors.toMap(Field::getName, Function.identity()));
    for (Map.Entry<byte[], byte[]> entry : row.getColumns().entrySet()) {
        String columnName = Bytes.toString(entry.getKey());
        if (!fieldMap.containsKey(columnName) || primaryKeys.contains(columnName)) {
            put.add(entry.getKey(), entry.getValue());
        } else {
            put.add(entry.getKey(), fieldToBytes(fieldMap.get(columnName)));
        }
    }
    return put;
}