Examples with DatasetId co.cask.cdap.proto.id.DatasetId used on opensource projects

Example 36 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class WorkflowDriver method deleteLocalDatasets.

private void deleteLocalDatasets() {
    for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) {
        final Map<String, String> datasetArguments = RuntimeArguments.extractScope(Scope.DATASET, entry.getKey(), basicWorkflowContext.getRuntimeArguments());
        if (Boolean.parseBoolean(datasetArguments.get("keep.local"))) {
            continue;
        }
        final String localInstanceName = entry.getValue();
        final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName);
        LOG.debug("Deleting Workflow local dataset instance: {}", localInstanceName);
        try {
            Retries.callWithRetries(new Retries.Callable<Void, Exception>() {

                @Override
                public Void call() throws Exception {
                    datasetFramework.deleteInstance(instanceId);
                    return null;
                }
            }, RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS));
        } catch (Exception e) {
            LOG.warn("Failed to delete the Workflow local dataset instance {}", localInstanceName, e);
        }
    }
}

Example 37 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetBasedStreamSizeScheduleStoreTest method testDeletion.

private void testDeletion(final ProgramId programId) throws Exception {
    final boolean defaultVersion = programId.getVersion().equals(ApplicationId.DEFAULT_VERSION);
    DatasetId storeTable = NamespaceId.SYSTEM.dataset(ScheduleStoreTableUtil.SCHEDULE_STORE_DATASET_NAME);
    final Table table = datasetFramework.getDataset(storeTable, ImmutableMap.<String, String>of(), null);
    Assert.assertNotNull(table);
    TransactionExecutor txnl = txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) table));
    final byte[] startKey = Bytes.toBytes(DatasetBasedStreamSizeScheduleStore.KEY_PREFIX);
    final byte[] stopKey = Bytes.stopKeyForPrefix(startKey);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Scanner scanner = table.scan(startKey, stopKey);
            Assert.assertNull(scanner.next());
            scanner.close();
        }
    });
    // Create one stream schedule - this will be persisted with new format
    scheduleStore.persist(programId, PROGRAM_TYPE, STREAM_SCHEDULE_1, MAP_1, 0L, 0L, 0L, 0L, true);
    // Create one stream schedule - based on the old format
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Create a programId without version so that we can create a old format schedule
            ProgramId defaultProgramId = new ProgramId(programId.getNamespace(), programId.getApplication(), programId.getType(), programId.getProgram());
            String newRowKey = scheduleStore.getRowKey(defaultProgramId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName());
            Row row = table.get(Bytes.toBytes(scheduleStore.getRowKey(programId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName())));
            Assert.assertFalse(row.isEmpty());
            byte[] oldRowKey = Bytes.toBytes(scheduleStore.removeAppVersion(newRowKey));
            for (Map.Entry<byte[], byte[]> entry : row.getColumns().entrySet()) {
                table.put(oldRowKey, entry.getKey(), entry.getValue());
            }
        }
    });
    // Make sure there are only two stream size schedules
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Scanner scanner = table.scan(startKey, stopKey);
            int numRows = 0;
            while (true) {
                Row row = scanner.next();
                if (row == null) {
                    break;
                }
                numRows++;
            }
            scanner.close();
            Assert.assertEquals(2, numRows);
        }
    });
    // This delete should have deleted both the old and new row format
    scheduleStore.delete(programId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName());
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Scanner scanner = table.scan(startKey, stopKey);
            if (defaultVersion) {
                Assert.assertNull(scanner.next());
            } else {
                Assert.assertNotNull(scanner.next());
                Assert.assertNull(scanner.next());
            }
            scanner.close();
        }
    });
    // If the version is not default, we need to delete the row which didn't have a version
    if (!defaultVersion) {
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // Create a programId without version so that we can create row key to delete the old format schedule
                ProgramId defaultProgramId = new ProgramId(programId.getNamespace(), programId.getApplication(), programId.getType(), programId.getProgram());
                String newRowKey = scheduleStore.getRowKey(defaultProgramId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName());
                byte[] oldRowKey = Bytes.toBytes(scheduleStore.removeAppVersion(newRowKey));
                Row row = table.get(oldRowKey);
                Assert.assertFalse(row.isEmpty());
                table.delete(oldRowKey);
            }
        });
    }
}

Example 38 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class TriggerCodecTest method testObjectContainingTrigger.

@Test
public void testObjectContainingTrigger() {
    ProgramSchedule proto1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), new ProtoTrigger.PartitionTrigger(new DatasetId("test1", "pdfs1"), 1), ImmutableList.<Constraint>of());
    ProgramSchedule sched1 = new ProgramSchedule("sched1", "one partition schedule", new NamespaceId("test").app("a").worker("ww"), ImmutableMap.of("prop3", "abc"), new PartitionTrigger(new DatasetId("test1", "pdfs1"), 1), ImmutableList.<Constraint>of());
    Assert.assertEquals(sched1, GSON.fromJson(GSON.toJson(proto1), ProgramSchedule.class));
    ProgramSchedule proto2 = new ProgramSchedule("schedone", "one time schedule", new NamespaceId("test3").app("abc").workflow("wf112"), ImmutableMap.of("prop", "all"), new ProtoTrigger.TimeTrigger("* * * 1 1"), ImmutableList.<Constraint>of());
    ProgramSchedule sched2 = new ProgramSchedule("schedone", "one time schedule", new NamespaceId("test3").app("abc").workflow("wf112"), ImmutableMap.of("prop", "all"), new TimeTrigger("* * * 1 1"), ImmutableList.<Constraint>of());
    Assert.assertEquals(sched2, GSON.fromJson(GSON.toJson(proto2), ProgramSchedule.class));
    ProgramSchedule proto3 = new ProgramSchedule("sched3", "one MB schedule", new NamespaceId("test3").app("abc").workflow("wf112"), ImmutableMap.of("prop", "all"), new ProtoTrigger.StreamSizeTrigger(new StreamId("x", "y"), 1), ImmutableList.<Constraint>of());
    ProgramSchedule sched3 = new ProgramSchedule("sched3", "one MB schedule", new NamespaceId("test3").app("abc").workflow("wf112"), ImmutableMap.of("prop", "all"), new StreamSizeTrigger(new StreamId("x", "y"), 1), ImmutableList.<Constraint>of());
    Assert.assertEquals(sched3, GSON.fromJson(GSON.toJson(proto3), ProgramSchedule.class));
}

Example 39 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetInstanceHandler method drop.

/**
   * Deletes a dataset instance, which also deletes the data owned by it.
   *
   * @param namespaceId namespace of the dataset instance
   * @param name name of the dataset instance
   * @throws Exception
   */
@DELETE
@Path("/data/datasets/{name}")
public void drop(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("name") String name) throws Exception {
    DatasetId instance = ConversionHelpers.toDatasetInstanceId(namespaceId, name);
    instanceService.drop(instance);
    responder.sendStatus(HttpResponseStatus.OK);
}

Example 40 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetInstanceService method create.

/**
   * Creates a dataset instance.
   *
   * @param namespaceId the namespace to create the dataset instance in
   * @param name the name of the new dataset instance
   * @param props the properties for the new dataset instance
   * @throws NamespaceNotFoundException if the specified namespace was not found
   * @throws DatasetAlreadyExistsException if a dataset with the same name already exists
   * @throws DatasetTypeNotFoundException if the dataset type was not found
   * @throws UnauthorizedException if perimeter security and authorization are enabled, and the current user does not
   *  have {@link Action#WRITE} privilege on the #instance's namespace
   */
void create(String namespaceId, String name, DatasetInstanceConfiguration props) throws Exception {
    NamespaceId namespace = ConversionHelpers.toNamespaceId(namespaceId);
    Principal principal = authenticationContext.getPrincipal();
    authorizationEnforcer.enforce(namespace, principal, Action.WRITE);
    ensureNamespaceExists(namespace);
    DatasetId datasetId = ConversionHelpers.toDatasetInstanceId(namespaceId, name);
    DatasetSpecification existing = instanceManager.get(datasetId);
    if (existing != null) {
        throw new DatasetAlreadyExistsException(datasetId);
    }
    DatasetTypeMeta typeMeta = getTypeInfo(namespace, props.getTypeName());
    if (typeMeta == null) {
        // Type not found in the instance's namespace and the system namespace. Bail out.
        throw new DatasetTypeNotFoundException(ConversionHelpers.toDatasetTypeId(namespace, props.getTypeName()));
    }
    // It is now determined that a new dataset will be created. First grant privileges, then create the dataset.
    // If creation fails, revoke the granted privileges. This ensures that just like delete, there may be orphaned
    // privileges in rare scenarios, but there can never be orphaned datasets.
    // If the dataset previously existed and was deleted, but revoking privileges somehow failed, there may be orphaned
    // privileges for the dataset. Revoke them first, so no users unintentionally get privileges on the dataset.
    privilegesManager.revoke(datasetId);
    // grant all privileges on the dataset to be created
    privilegesManager.grant(datasetId, principal, EnumSet.allOf(Action.class));
    LOG.info("Creating dataset {}.{}, type name: {}, properties: {}", namespaceId, name, props.getTypeName(), props.getProperties());
    // Note how we execute configure() via opExecutorClient (outside of ds service) to isolate running user code
    try {
        String ownerPrincipal = props.getOwnerPrincipal();
        // exists or not
        if (ownerPrincipal != null) {
            KerberosPrincipalId owner = new KerberosPrincipalId(ownerPrincipal);
            ownerAdmin.add(datasetId, owner);
        }
        try {
            DatasetSpecification spec = opExecutorClient.create(datasetId, typeMeta, DatasetProperties.builder().addAll(props.getProperties()).setDescription(props.getDescription()).build());
            instanceManager.add(namespace, spec);
            metaCache.invalidate(datasetId);
            publishAudit(datasetId, AuditType.CREATE);
            // Enable explore
            enableExplore(datasetId, spec, props);
        } catch (Exception e) {
            // there was a problem in creating the dataset instance so delete the owner if it got added earlier
            // safe to call for entities which does not have an owner too
            ownerAdmin.delete(datasetId);
            throw e;
        }
    } catch (Exception e) {
        // there was a problem in creating the dataset instance so revoke the privileges
        privilegesManager.revoke(datasetId);
        throw e;
    }
}