Examples with Update io.cdap.cdap.data2.dataset2.lib.table.Update used on opensource projects

Example 26 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class InMemoryTableService method merge.

private static void merge(ConcurrentNavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, Update>>> table, byte[] row, Map<byte[], Update> changes, long version) {
    // get the correct row from the table, create it if it doesn't exist
    NavigableMap<byte[], NavigableMap<Long, Update>> rowMap = table.get(row);
    if (rowMap == null) {
        rowMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
        table.put(row, rowMap);
    }
    // now merge the changes into the row, one by one
    for (Map.Entry<byte[], Update> keyVal : changes.entrySet()) {
        // create the column in the row if it does not exist
        NavigableMap<Long, Update> colMap = rowMap.get(keyVal.getKey());
        if (colMap == null) {
            colMap = Maps.newTreeMap();
            rowMap.put(keyVal.getKey(), colMap);
        }
        // put into the column with given version
        Update merged = Updates.mergeUpdates(colMap.get(version), keyVal.getValue());
        colMap.put(version, merged);
    }
}

Example 27 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class ProgramScheduleStoreDatasetTest method testFindSchedulesByEventAndUpdateSchedule.

@Test
public void testFindSchedulesByEventAndUpdateSchedule() throws Exception {
    DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class);
    TransactionSystemClient txClient = getInjector().getInstance(TransactionSystemClient.class);
    TransactionExecutorFactory txExecutorFactory = new DynamicTransactionExecutorFactory(txClient);
    dsFramework.truncateInstance(Schedulers.STORE_DATASET_ID);
    final ProgramScheduleStoreDataset store = dsFramework.getDataset(Schedulers.STORE_DATASET_ID, new HashMap<String, String>(), null);
    Assert.assertNotNull(store);
    TransactionExecutor txExecutor = txExecutorFactory.createExecutor(Collections.singleton((TransactionAware) store));
    final ProgramSchedule sched11 = new ProgramSchedule("sched11", "one partition schedule", PROG1_ID, ImmutableMap.of("prop3", "abc"), new PartitionTrigger(DS1_ID, 1), ImmutableList.<Constraint>of());
    final ProgramSchedule sched12 = new ProgramSchedule("sched12", "two partition schedule", PROG1_ID, ImmutableMap.of("propper", "popper"), new PartitionTrigger(DS2_ID, 2), ImmutableList.<Constraint>of());
    final ProgramSchedule sched22 = new ProgramSchedule("sched22", "twentytwo partition schedule", PROG2_ID, ImmutableMap.of("nn", "4"), new PartitionTrigger(DS2_ID, 22), ImmutableList.<Constraint>of());
    final ProgramSchedule sched31 = new ProgramSchedule("sched31", "a program status trigger", PROG3_ID, ImmutableMap.of("propper", "popper"), new ProgramStatusTrigger(PROG1_ID, ProgramStatus.COMPLETED, ProgramStatus.FAILED, ProgramStatus.KILLED), ImmutableList.<Constraint>of());
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // event for DS1 or DS2 should trigger nothing. validate it returns an empty collection
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForPartition(DS1_ID)).isEmpty());
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForPartition(DS2_ID)).isEmpty());
            // event for PROG1 should trigger nothing. it should also return an empty collection
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.COMPLETED)).isEmpty());
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.FAILED)).isEmpty());
            Assert.assertTrue(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.KILLED)).isEmpty());
        }
    });
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            store.addSchedules(ImmutableList.of(sched11, sched12, sched22, sched31));
        }
    });
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // event for ProgramStatus should trigger only sched31
            Assert.assertEquals(ImmutableSet.of(sched31), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.COMPLETED))));
            Assert.assertEquals(ImmutableSet.of(sched31), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.FAILED))));
            Assert.assertEquals(ImmutableSet.of(sched31), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.KILLED))));
            // event for DS1 should trigger only sched11
            Assert.assertEquals(ImmutableSet.of(sched11), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS1_ID))));
            // event for DS2 triggers only sched12 and sched22
            Assert.assertEquals(ImmutableSet.of(sched12, sched22), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS2_ID))));
        }
    });
    final ProgramSchedule sched11New = new ProgramSchedule(sched11.getName(), "time schedule", PROG1_ID, ImmutableMap.of("timeprop", "time"), new TimeTrigger("* * * * *"), ImmutableList.<Constraint>of());
    final ProgramSchedule sched12New = new ProgramSchedule(sched12.getName(), "one partition schedule", PROG1_ID, ImmutableMap.of("pp", "p"), new PartitionTrigger(DS1_ID, 2), ImmutableList.<Constraint>of());
    final ProgramSchedule sched22New = new ProgramSchedule(sched22.getName(), "program3 failed schedule", PROG2_ID, ImmutableMap.of("ss", "s"), new ProgramStatusTrigger(PROG3_ID, ProgramStatus.FAILED), ImmutableList.<Constraint>of());
    final ProgramSchedule sched31New = new ProgramSchedule(sched31.getName(), "program1 failed schedule", PROG3_ID, ImmutableMap.of("abcd", "efgh"), new ProgramStatusTrigger(PROG1_ID, ProgramStatus.FAILED), ImmutableList.<Constraint>of());
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            store.updateSchedule(sched11New);
            store.updateSchedule(sched12New);
            store.updateSchedule(sched22New);
            store.updateSchedule(sched31New);
        }
    });
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // event for DS1 should trigger only sched12New after update
            Assert.assertEquals(ImmutableSet.of(sched12New), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS1_ID))));
            // event for DS2 triggers no schedule after update
            Assert.assertEquals(ImmutableSet.<ProgramSchedule>of(), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForPartition(DS2_ID))));
            // event for PS triggers only for failed program statuses, not completed nor killed
            Assert.assertEquals(ImmutableSet.of(sched31New), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.FAILED))));
            Assert.assertEquals(ImmutableSet.of(), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.COMPLETED))));
            Assert.assertEquals(ImmutableSet.of(), toScheduleSet(store.findSchedules(Schedulers.triggerKeyForProgramStatus(PROG1_ID, ProgramStatus.KILLED))));
        }
    });
}

Example 28 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class DatasetInstanceService method update.

/**
 * Updates an existing Dataset specification properties.
 * {@link DatasetInstanceConfiguration} is constructed based on request and the Dataset instance is updated.
 *
 * @param instance the dataset instance
 * @param properties the dataset properties to be used
 * @throws NamespaceNotFoundException if the specified namespace was not found
 * @throws DatasetNotFoundException if the dataset was not found
 * @throws DatasetTypeNotFoundException if the type of the existing dataset was not found
 * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
 *  have {@link StandardPermission#UPDATE} privilege on the #instance
 */
void update(DatasetId instance, Map<String, String> properties) throws Exception {
    ensureNamespaceExists(instance.getParent());
    Principal requestingUser = authenticationContext.getPrincipal();
    if (!DatasetsUtil.isSystemDatasetInUserNamespace(instance)) {
        accessEnforcer.enforce(instance, requestingUser, StandardPermission.UPDATE);
    }
    DatasetSpecification existing = instanceManager.get(instance);
    if (existing == null) {
        throw new DatasetNotFoundException(instance);
    }
    LOG.info("Update dataset {}, properties: {}", instance.getEntityName(), ConversionHelpers.toJson(properties));
    // by pass the auth check for dataset type when the operation is not creation
    DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), existing.getType(), true);
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(instance.getParent(), existing.getType()));
    }
    // Note how we execute configure() via opExecutorClient (outside of ds service) to isolate running user code
    DatasetProperties datasetProperties = DatasetProperties.of(properties);
    DatasetCreationResponse response = opExecutorClient.update(instance, typeMeta, datasetProperties, existing);
    DatasetSpecification spec = response.getSpec();
    instanceManager.add(instance.getParent(), spec);
    metaCache.invalidate(instance);
    updateExplore(instance, datasetProperties, existing, spec);
    publishAudit(instance, AuditType.UPDATE);
    publishMetadata(instance, response.getMetadata());
}

Example 29 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class DatasetInstanceService method create.

/**
 * Creates a dataset instance.
 *
 * @param namespaceId the namespace to create the dataset instance in
 * @param name the name of the new dataset instance
 * @param props the properties for the new dataset instance
 * @throws NamespaceNotFoundException if the specified namespace was not found
 * @throws DatasetAlreadyExistsException if a dataset with the same name already exists
 * @throws DatasetTypeNotFoundException if the dataset type was not found
 * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
 *  have {@link StandardPermission#UPDATE} privilege on the #instance's namespace
 */
void create(String namespaceId, String name, DatasetInstanceConfiguration props) throws Exception {
    NamespaceId namespace = ConversionHelpers.toNamespaceId(namespaceId);
    DatasetId datasetId = ConversionHelpers.toDatasetInstanceId(namespaceId, name);
    Principal requestingUser = authenticationContext.getPrincipal();
    String ownerPrincipal = props.getOwnerPrincipal();
    // need to enforce on the principal id if impersonation is involved
    KerberosPrincipalId effectiveOwner = SecurityUtil.getEffectiveOwner(ownerAdmin, namespace, ownerPrincipal);
    if (DatasetsUtil.isUserDataset(datasetId)) {
        LOG.trace("Authorizing impersonation for dataset {}", name);
        if (effectiveOwner != null) {
            accessEnforcer.enforce(effectiveOwner, requestingUser, AccessPermission.SET_OWNER);
        }
        accessEnforcer.enforce(datasetId, requestingUser, StandardPermission.CREATE);
        LOG.trace("Authorized impersonation for dataset {}", name);
    }
    LOG.trace("Ensuring existence of namespace {} for dataset {}", namespace, name);
    ensureNamespaceExists(namespace);
    LOG.trace("Ensured existence of namespace {} for dataset {}", namespace, name);
    LOG.trace("Retrieving instance metadata from MDS for dataset {}", name);
    DatasetSpecification existing = instanceManager.get(datasetId);
    if (existing != null) {
        throw new DatasetAlreadyExistsException(datasetId);
    }
    LOG.trace("Retrieved instance metadata from MDS for dataset {}", name);
    // for creation, we need enforcement for dataset type for user dataset, but bypass for system datasets
    DatasetTypeMeta typeMeta = getTypeInfo(namespace, props.getTypeName(), !DatasetsUtil.isUserDataset(datasetId));
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(namespace, props.getTypeName()));
    }
    LOG.info("Creating dataset {}.{}, type name: {}, properties: {}", namespaceId, name, props.getTypeName(), props.getProperties());
    // exists or not
    if (ownerPrincipal != null) {
        LOG.trace("Adding owner for dataset {}", name);
        KerberosPrincipalId owner = new KerberosPrincipalId(ownerPrincipal);
        ownerAdmin.add(datasetId, owner);
        LOG.trace("Added owner {} for dataset {}", owner, name);
    }
    try {
        DatasetProperties datasetProperties = DatasetProperties.builder().addAll(props.getProperties()).setDescription(props.getDescription()).build();
        LOG.trace("Calling op executor service to configure dataset {}", name);
        DatasetCreationResponse response = opExecutorClient.create(datasetId, typeMeta, datasetProperties);
        LOG.trace("Received spec and metadata from op executor service for dataset {}: {}", name, response);
        LOG.trace("Adding instance metadata for dataset {}", name);
        DatasetSpecification spec = response.getSpec();
        instanceManager.add(namespace, spec);
        LOG.trace("Added instance metadata for dataset {}", name);
        metaCache.invalidate(datasetId);
        LOG.trace("Publishing audit for creation of dataset {}", name);
        publishAudit(datasetId, AuditType.CREATE);
        LOG.trace("Published audit for creation of dataset {}", name);
        SystemMetadata metadata = response.getMetadata();
        LOG.trace("Publishing system metadata for creation of dataset {}: {}", name, metadata);
        publishMetadata(datasetId, metadata);
        LOG.trace("Published system metadata for creation of dataset {}", name);
        // Enable explore
        enableExplore(datasetId, spec, props);
    } catch (Exception e) {
        // there was a problem in creating the dataset instance so delete the owner if it got added earlier
        // safe to call for entities which does not have an owner too
        ownerAdmin.delete(datasetId);
        throw e;
    }
}

Example 30 with Update

use of io.cdap.cdap.data2.dataset2.lib.table.Update in project cdap by caskdata.

the class InMemoryTableService method merge.

private static void merge(ConcurrentNavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, Update>>> table, byte[] row, Map<byte[], Update> changes, long version) {
    // get the correct row from the table, create it if it doesn't exist
    NavigableMap<byte[], NavigableMap<Long, Update>> rowMap = table.get(row);
    if (rowMap == null) {
        rowMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
        table.put(row, rowMap);
    }
    // now merge the changes into the row, one by one
    for (Map.Entry<byte[], Update> keyVal : changes.entrySet()) {
        // create the column in the row if it does not exist
        NavigableMap<Long, Update> colMap = rowMap.get(keyVal.getKey());
        if (colMap == null) {
            colMap = Maps.newTreeMap();
            rowMap.put(keyVal.getKey(), colMap);
        }
        // put into the column with given version
        Update merged = Updates.mergeUpdates(colMap.get(version), keyVal.getValue());
        colMap.put(version, merged);
    }
}