Examples with DatasetId io.cdap.cdap.proto.id.DatasetId used on opensource projects

Example 11 with DatasetId

use of io.cdap.cdap.proto.id.DatasetId in project cdap by caskdata.

the class FieldLineageAdmin method convertSummaryToDatasetField.

private Set<DatasetField> convertSummaryToDatasetField(Set<EndPointField> summary) {
    Map<DatasetId, Set<String>> endPointFields = convertSummaryToDatasetMap(summary);
    Set<DatasetField> result = new HashSet<>();
    for (Map.Entry<DatasetId, Set<String>> entry : endPointFields.entrySet()) {
        result.add(new DatasetField(entry.getKey(), entry.getValue()));
    }
    return result;
}

Example 12 with DatasetId

use of io.cdap.cdap.proto.id.DatasetId in project cdap by caskdata.

the class UsageTable method getDatasetsFromPrefix.

private Set<DatasetId> getDatasetsFromPrefix(List<Field<?>> prefix) throws IOException {
    Set<DatasetId> datasets = new HashSet<>();
    try (CloseableIterator<StructuredRow> iterator = table.scan(Range.singleton(prefix), Integer.MAX_VALUE)) {
        while (iterator.hasNext()) {
            StructuredRow row = iterator.next();
            datasets.add(GSON.fromJson(row.getString(StoreDefinition.UsageStore.DATASET_FIELD), DatasetKey.class).getDataset());
        }
    }
    return datasets;
}

Example 13 with DatasetId

use of io.cdap.cdap.proto.id.DatasetId in project cdap by caskdata.

the class HBaseDatasetMetricsReporter method report.

private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException, UnauthorizedException {
    Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap();
    for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) {
        String hbaseNamespace = statEntry.getKey().getNamespace();
        String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace);
        // tableNames that doesn't start with user are ignored
        if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) {
            continue;
        }
        String tableName = statEntry.getKey().getTableName();
        try {
            Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace));
            for (DatasetSpecificationSummary spec : instances) {
                DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName()));
                if (specification.isParent(tableName)) {
                    MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName()));
                    collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB());
                    break;
                }
            }
        } catch (DatasetManagementException | ServiceUnavailableException e) {
        // No op
        }
    }
}

Example 14 with DatasetId

use of io.cdap.cdap.proto.id.DatasetId in project cdap by caskdata.

the class LevelDBDatasetMetricsReporter method report.

private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat) throws DatasetManagementException, UnauthorizedException {
    for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) {
        String namespace = statEntry.getKey().getNamespace();
        // emit metrics for only user datasets, tables in system namespace are ignored
        if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) {
            continue;
        }
        String tableName = statEntry.getKey().getTableName();
        Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(namespace));
        for (DatasetSpecificationSummary spec : instances) {
            DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(namespace, spec.getName()));
            if (specification.isParent(tableName)) {
                MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, namespace, Constants.Metrics.Tag.DATASET, spec.getName()));
                int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB);
                collector.gauge("dataset.size.mb", sizeInMb);
                break;
            }
        }
    }
}

Example 15 with DatasetId

use of io.cdap.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetInstanceHandlerTest method testNotFound.

@Test
public void testNotFound() throws IOException {
    NamespaceId nonExistent = new NamespaceId("nonexistent");
    DatasetId datasetInstance = nonExistent.dataset("ds");
    HttpResponse response = makeInstancesRequest(nonExistent.getNamespace());
    assertNamespaceNotFound(response, nonExistent);
    // TODO: commented out for now until we add back namespace checks on get dataset CDAP-3901
    // response = getInstance(datasetInstance);
    // assertNamespaceNotFound(response, nonExistent);
    response = createInstance(datasetInstance, Table.class.getName(), null);
    assertNamespaceNotFound(response, nonExistent);
    response = updateInstance(datasetInstance, DatasetProperties.EMPTY);
    assertNamespaceNotFound(response, nonExistent);
    response = deleteInstance(datasetInstance);
    assertNamespaceNotFound(response, nonExistent);
    // it should be ok to use system
    response = getInstances(NamespaceId.SYSTEM.getNamespace());
    Assert.assertEquals(200, response.getResponseCode());
}