Examples with DatasetSpecificationSummary io.cdap.cdap.proto.DatasetSpecificationSummary used on opensource projects

Example 1 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class LocalDatasetDeleterRunnable method run.

@Override
public void run() {
    try {
        List<NamespaceMeta> list = namespaceAdmin.list();
        for (NamespaceMeta namespaceMeta : list) {
            Collection<DatasetSpecificationSummary> specs = datasetFramework.getInstances(namespaceMeta.getNamespaceId(), PROPERTIES);
            if (specs.isEmpty()) {
                // avoid fetching run records
                continue;
            }
            Set<String> activeRuns = getActiveRuns(namespaceMeta.getNamespaceId());
            for (DatasetSpecificationSummary spec : specs) {
                deleteLocalDataset(namespaceMeta.getName(), spec.getName(), activeRuns, spec.getProperties());
            }
        }
    } catch (Throwable t) {
        LOG.warn("Failed to delete the local datasets.", t);
    }
}

Example 2 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class DatasetInstanceHandlerTest method testUpdateInstance.

@Test
public void testUpdateInstance() throws Exception {
    // nothing has been created, modules and types list is empty
    List<DatasetSpecificationSummary> instances = getInstances().getResponseObject();
    // nothing in the beginning
    Assert.assertEquals(0, instances.size());
    try {
        DatasetProperties props = DatasetProperties.builder().add("prop1", "val1").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
        // deploy modules
        deployModule("module1", TestModule1.class);
        deployModule("module2", TestModule2.class);
        // create dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, createInstance("dataset1", "datasetType2", props).getResponseCode());
        // verify instance was created
        instances = getInstances().getResponseObject();
        Assert.assertEquals(1, instances.size());
        DatasetMeta meta = getInstanceObject("dataset1").getResponseObject();
        Map<String, String> storedOriginalProps = meta.getSpec().getOriginalProperties();
        Assert.assertEquals(props.getProperties(), storedOriginalProps);
        Map<String, String> retrievedProps = getInstanceProperties("dataset1").getResponseObject();
        Assert.assertEquals(props.getProperties(), retrievedProps);
        // these properties are incompatible because TestModule1.NOT_RECONFIGURABLE may not change
        DatasetProperties newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "that").build();
        Assert.assertEquals(HttpStatus.SC_CONFLICT, updateInstance("dataset1", newProps).getResponseCode());
        // update dataset instance with valid properties
        newProps = DatasetProperties.builder().add("prop2", "val2").add(TestModule2.NOT_RECONFIGURABLE, "this").build();
        Assert.assertEquals(HttpStatus.SC_OK, updateInstance("dataset1", newProps).getResponseCode());
        meta = getInstanceObject("dataset1").getResponseObject();
        Assert.assertEquals(newProps.getProperties(), meta.getSpec().getOriginalProperties());
        Assert.assertEquals("val2", meta.getSpec().getProperty("prop2"));
        Assert.assertNull(meta.getSpec().getProperty("prop1"));
        retrievedProps = getInstanceProperties("dataset1").getResponseObject();
        Assert.assertEquals(newProps.getProperties(), retrievedProps);
    } finally {
        // delete dataset instance
        Assert.assertEquals(HttpStatus.SC_OK, deleteInstance("dataset1").getResponseCode());
        Assert.assertEquals(0, getInstances().getResponseObject().size());
        // delete dataset modules
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module2").getResponseCode());
        Assert.assertEquals(HttpStatus.SC_OK, deleteModule("module1").getResponseCode());
    }
}

Example 3 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class ConversionHelpers method spec2Summary.

static Collection<DatasetSpecificationSummary> spec2Summary(Collection<DatasetSpecification> specs) {
    List<DatasetSpecificationSummary> datasetSummaries = Lists.newArrayList();
    for (DatasetSpecification spec : specs) {
        // TODO: (CDAP-3097) handle system datasets specially within a namespace instead of filtering them out
        // by the handler. This filter is only in the list endpoint because the other endpoints are used by
        // HBaseQueueAdmin through DatasetFramework.
        spec = DatasetsUtil.fixOriginalProperties(spec);
        datasetSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getDescription(), spec.getOriginalProperties()));
    }
    return datasetSummaries;
}

Example 4 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class InMemoryDatasetFramework method getInstances.

@Override
public Collection<DatasetSpecificationSummary> getInstances(NamespaceId namespaceId, Map<String, String> properties) {
    readLock.lock();
    try {
        // don't expect this to be called a lot.
        // might be better to maintain this collection separately and just return it, but seems like its not worth it.
        Collection<DatasetSpecification> specs = instances.row(namespaceId).values();
        ImmutableList.Builder<DatasetSpecificationSummary> specSummaries = ImmutableList.builder();
        for (DatasetSpecification spec : specs) {
            if (properties.isEmpty() || Maps.difference(properties, spec.getProperties()).entriesOnlyOnLeft().isEmpty()) {
                specSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getProperties()));
            }
        }
        return specSummaries.build();
    } finally {
        readLock.unlock();
    }
}

Example 5 with DatasetSpecificationSummary

use of io.cdap.cdap.proto.DatasetSpecificationSummary in project cdap by caskdata.

the class DatasetClient method list.

/**
 * Lists all datasets.
 *
 * @return list of {@link DatasetSpecificationSummary}.
 * @throws IOException if a network error occurred
 * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server
 */
public List<DatasetSpecificationSummary> list(NamespaceId namespace) throws IOException, UnauthenticatedException, UnauthorizedException {
    URL url = config.resolveNamespacedURLV3(namespace, "data/datasets");
    HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken());
    return ObjectResponse.fromJsonBody(response, new TypeToken<List<DatasetSpecificationSummary>>() {
    }).getResponseObject();
}