use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class LineageTestRun method testFlowLineage.
@Test
public void testFlowLineage() throws Exception {
NamespaceId namespace = new NamespaceId("testFlowLineage");
ApplicationId app = namespace.app(AllProgramsApp.NAME);
ProgramId flow = app.flow(AllProgramsApp.NoOpFlow.NAME);
DatasetId dataset = namespace.dataset(AllProgramsApp.DATASET_NAME);
StreamId stream = namespace.stream(AllProgramsApp.STREAM_NAME);
namespaceClient.create(new NamespaceMeta.Builder().setName(namespace).build());
try {
appClient.deploy(namespace, createAppJarFile(AllProgramsApp.class));
// Add metadata to applicaton
ImmutableMap<String, String> appProperties = ImmutableMap.of("app-key1", "app-value1");
addProperties(app, appProperties);
Assert.assertEquals(appProperties, getProperties(app, MetadataScope.USER));
ImmutableSet<String> appTags = ImmutableSet.of("app-tag1");
addTags(app, appTags);
Assert.assertEquals(appTags, getTags(app, MetadataScope.USER));
// Add metadata to flow
ImmutableMap<String, String> flowProperties = ImmutableMap.of("flow-key1", "flow-value1");
addProperties(flow, flowProperties);
Assert.assertEquals(flowProperties, getProperties(flow, MetadataScope.USER));
ImmutableSet<String> flowTags = ImmutableSet.of("flow-tag1", "flow-tag2");
addTags(flow, flowTags);
Assert.assertEquals(flowTags, getTags(flow, MetadataScope.USER));
// Add metadata to dataset
ImmutableMap<String, String> dataProperties = ImmutableMap.of("data-key1", "data-value1");
addProperties(dataset, dataProperties);
Assert.assertEquals(dataProperties, getProperties(dataset, MetadataScope.USER));
ImmutableSet<String> dataTags = ImmutableSet.of("data-tag1", "data-tag2");
addTags(dataset, dataTags);
Assert.assertEquals(dataTags, getTags(dataset, MetadataScope.USER));
// Add metadata to stream
ImmutableMap<String, String> streamProperties = ImmutableMap.of("stream-key1", "stream-value1");
addProperties(stream, streamProperties);
Assert.assertEquals(streamProperties, getProperties(stream, MetadataScope.USER));
ImmutableSet<String> streamTags = ImmutableSet.of("stream-tag1", "stream-tag2");
addTags(stream, streamTags);
Assert.assertEquals(streamTags, getTags(stream, MetadataScope.USER));
long startTime = TimeMathParser.nowInSeconds();
RunId flowRunId = runAndWait(flow);
// Wait for few seconds so that the stop time secs is more than start time secs.
TimeUnit.SECONDS.sleep(2);
waitForStop(flow, true);
long stopTime = TimeMathParser.nowInSeconds();
// Fetch dataset lineage
LineageRecord lineage = fetchLineage(dataset, startTime, stopTime, 10);
LineageRecord expected = LineageSerializer.toLineageRecord(startTime, stopTime, new Lineage(ImmutableSet.of(new Relation(dataset, flow, AccessType.UNKNOWN, flowRunId, ImmutableSet.of(flow.flowlet(AllProgramsApp.A.NAME))), new Relation(stream, flow, AccessType.READ, flowRunId, ImmutableSet.of(flow.flowlet(AllProgramsApp.A.NAME))))), Collections.<CollapseType>emptySet());
Assert.assertEquals(expected, lineage);
// Fetch dataset lineage with time strings
lineage = fetchLineage(dataset, "now-1h", "now+1h", 10);
Assert.assertEquals(expected.getRelations(), lineage.getRelations());
// Fetch stream lineage
lineage = fetchLineage(stream, startTime, stopTime, 10);
// same as dataset's lineage
Assert.assertEquals(expected, lineage);
// Fetch stream lineage with time strings
lineage = fetchLineage(stream, "now-1h", "now+1h", 10);
// same as dataset's lineage
Assert.assertEquals(expected.getRelations(), lineage.getRelations());
// Assert metadata
// Id.Flow needs conversion to Id.Program JIRA - CDAP-3658
Assert.assertEquals(toSet(new MetadataRecord(app, MetadataScope.USER, appProperties, appTags), new MetadataRecord(flow, MetadataScope.USER, flowProperties, flowTags), new MetadataRecord(dataset, MetadataScope.USER, dataProperties, dataTags), new MetadataRecord(stream, MetadataScope.USER, streamProperties, streamTags)), fetchRunMetadata(flow.run(flowRunId.getId())));
// Assert with a time range after the flow run should return no results
long laterStartTime = stopTime + 1000;
long laterEndTime = stopTime + 5000;
// Fetch stream lineage
lineage = fetchLineage(stream, laterStartTime, laterEndTime, 10);
Assert.assertEquals(LineageSerializer.toLineageRecord(laterStartTime, laterEndTime, new Lineage(ImmutableSet.<Relation>of()), Collections.<CollapseType>emptySet()), lineage);
// Assert with a time range before the flow run should return no results
long earlierStartTime = startTime - 5000;
long earlierEndTime = startTime - 1000;
// Fetch stream lineage
lineage = fetchLineage(stream, earlierStartTime, earlierEndTime, 10);
Assert.assertEquals(LineageSerializer.toLineageRecord(earlierStartTime, earlierEndTime, new Lineage(ImmutableSet.<Relation>of()), Collections.<CollapseType>emptySet()), lineage);
// Test bad time ranges
fetchLineage(dataset, "sometime", "sometime", 10, BadRequestException.class);
fetchLineage(dataset, "now+1h", "now-1h", 10, BadRequestException.class);
// Test non-existent run
assertRunMetadataNotFound(flow.run(RunIds.generate(1000).getId()));
} finally {
namespaceClient.delete(namespace);
}
}
use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class LineageTestRun method testAllProgramsLineage.
@Test
public void testAllProgramsLineage() throws Exception {
NamespaceId namespace = new NamespaceId("testAllProgramsLineage");
ApplicationId app = namespace.app(AllProgramsApp.NAME);
ProgramId flow = app.flow(AllProgramsApp.NoOpFlow.NAME);
ProgramId mapreduce = app.mr(AllProgramsApp.NoOpMR.NAME);
ProgramId mapreduce2 = app.mr(AllProgramsApp.NoOpMR2.NAME);
ProgramId spark = app.spark(AllProgramsApp.NoOpSpark.NAME);
ProgramId service = app.service(AllProgramsApp.NoOpService.NAME);
ProgramId worker = app.worker(AllProgramsApp.NoOpWorker.NAME);
ProgramId workflow = app.workflow(AllProgramsApp.NoOpWorkflow.NAME);
DatasetId dataset = namespace.dataset(AllProgramsApp.DATASET_NAME);
DatasetId dataset2 = namespace.dataset(AllProgramsApp.DATASET_NAME2);
DatasetId dataset3 = namespace.dataset(AllProgramsApp.DATASET_NAME3);
StreamId stream = namespace.stream(AllProgramsApp.STREAM_NAME);
namespaceClient.create(new NamespaceMeta.Builder().setName(namespace.getNamespace()).build());
try {
appClient.deploy(namespace, createAppJarFile(AllProgramsApp.class));
// Add metadata
ImmutableSet<String> sparkTags = ImmutableSet.of("spark-tag1", "spark-tag2");
addTags(spark, sparkTags);
Assert.assertEquals(sparkTags, getTags(spark, MetadataScope.USER));
ImmutableSet<String> workerTags = ImmutableSet.of("worker-tag1");
addTags(worker, workerTags);
Assert.assertEquals(workerTags, getTags(worker, MetadataScope.USER));
ImmutableMap<String, String> datasetProperties = ImmutableMap.of("data-key1", "data-value1");
addProperties(dataset, datasetProperties);
Assert.assertEquals(datasetProperties, getProperties(dataset, MetadataScope.USER));
// Start all programs
RunId flowRunId = runAndWait(flow);
RunId mrRunId = runAndWait(mapreduce);
RunId mrRunId2 = runAndWait(mapreduce2);
RunId sparkRunId = runAndWait(spark);
runAndWait(workflow);
RunId workflowMrRunId = getRunId(mapreduce, mrRunId);
RunId serviceRunId = runAndWait(service);
// Worker makes a call to service to make it access datasets,
// hence need to make sure service starts before worker, and stops after it.
RunId workerRunId = runAndWait(worker);
// Wait for programs to finish
waitForStop(flow, true);
waitForStop(mapreduce, false);
waitForStop(mapreduce2, false);
waitForStop(spark, false);
waitForStop(workflow, false);
waitForStop(worker, false);
waitForStop(service, true);
long now = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
long oneHour = TimeUnit.HOURS.toSeconds(1);
// Fetch dataset lineage
LineageRecord lineage = fetchLineage(dataset, now - oneHour, now + oneHour, toSet(CollapseType.ACCESS), 10);
// dataset is accessed by all programs
LineageRecord expected = LineageSerializer.toLineageRecord(now - oneHour, now + oneHour, new Lineage(ImmutableSet.of(// Dataset access
new Relation(dataset, flow, AccessType.UNKNOWN, flowRunId, toSet(flow.flowlet(AllProgramsApp.A.NAME))), new Relation(dataset, mapreduce, AccessType.WRITE, mrRunId), new Relation(dataset, mapreduce2, AccessType.WRITE, mrRunId2), new Relation(dataset2, mapreduce2, AccessType.READ, mrRunId2), new Relation(dataset, spark, AccessType.READ, sparkRunId), new Relation(dataset2, spark, AccessType.WRITE, sparkRunId), new Relation(dataset3, spark, AccessType.READ, sparkRunId), new Relation(dataset3, spark, AccessType.WRITE, sparkRunId), new Relation(dataset, mapreduce, AccessType.WRITE, workflowMrRunId), new Relation(dataset, service, AccessType.WRITE, serviceRunId), new Relation(dataset, worker, AccessType.WRITE, workerRunId), // Stream access
new Relation(stream, flow, AccessType.READ, flowRunId, ImmutableSet.of(flow.flowlet(AllProgramsApp.A.NAME))), new Relation(stream, mapreduce, AccessType.READ, mrRunId), new Relation(stream, spark, AccessType.READ, sparkRunId), new Relation(stream, mapreduce, AccessType.READ, workflowMrRunId), new Relation(stream, worker, AccessType.WRITE, workerRunId))), toSet(CollapseType.ACCESS));
Assert.assertEquals(expected, lineage);
// Fetch stream lineage
lineage = fetchLineage(stream, now - oneHour, now + oneHour, toSet(CollapseType.ACCESS), 10);
// stream too is accessed by all programs
Assert.assertEquals(expected, lineage);
// Assert metadata
// Id.Flow needs conversion to Id.Program JIRA - CDAP-3658
Assert.assertEquals(toSet(new MetadataRecord(app, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(flow, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(dataset, MetadataScope.USER, datasetProperties, emptySet()), new MetadataRecord(stream, MetadataScope.USER, emptyMap(), emptySet())), fetchRunMetadata(flow.run(flowRunId.getId())));
// Id.Worker needs conversion to Id.Program JIRA - CDAP-3658
ProgramId programForWorker = new ProgramId(worker.getNamespace(), worker.getApplication(), worker.getType(), worker.getEntityName());
Assert.assertEquals(toSet(new MetadataRecord(app, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(programForWorker, MetadataScope.USER, emptyMap(), workerTags), new MetadataRecord(dataset, MetadataScope.USER, datasetProperties, emptySet()), new MetadataRecord(stream, MetadataScope.USER, emptyMap(), emptySet())), fetchRunMetadata(worker.run(workerRunId.getId())));
// Id.Spark needs conversion to Id.Program JIRA - CDAP-3658
ProgramId programForSpark = new ProgramId(spark.getNamespace(), spark.getApplication(), spark.getType(), spark.getEntityName());
Assert.assertEquals(toSet(new MetadataRecord(app, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(programForSpark, MetadataScope.USER, emptyMap(), sparkTags), new MetadataRecord(dataset, MetadataScope.USER, datasetProperties, emptySet()), new MetadataRecord(dataset2, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(dataset3, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(stream, MetadataScope.USER, emptyMap(), emptySet())), fetchRunMetadata(spark.run(sparkRunId.getId())));
} finally {
namespaceClient.delete(namespace);
}
}
use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class HBaseQueueProducer method persist.
/**
* Persist queue entries into HBase.
*/
@Override
protected int persist(Iterable<QueueEntry> entries, Transaction transaction) throws IOException {
int count = 0;
List<Put> puts = Lists.newArrayList();
int bytes = 0;
List<byte[]> rowKeys = Lists.newArrayList();
long writePointer = transaction.getWritePointer();
ensureValidTxLifetime(writePointer);
for (QueueEntry entry : entries) {
rowKeys.clear();
queueStrategy.getRowKeys(consumerGroupConfigs, entry, queueRowPrefix, writePointer, count, rowKeys);
rollbackKeys.addAll(rowKeys);
byte[] metaData = QueueEntry.serializeHashKeys(entry.getHashKeys());
for (byte[] rowKey : rowKeys) {
// No need to write ts=writePointer, as the row key already contains the writePointer
Put put = new Put(rowKey);
put.add(QueueEntryRow.COLUMN_FAMILY, QueueEntryRow.DATA_COLUMN, entry.getData());
put.add(QueueEntryRow.COLUMN_FAMILY, QueueEntryRow.META_COLUMN, metaData);
puts.add(put);
bytes += entry.getData().length;
}
count++;
}
hTable.put(puts);
hTable.flushCommits();
return bytes;
}
use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class HBaseTableCoprocessorTestRun method setupBeforeClass.
@BeforeClass
public static void setupBeforeClass() throws Exception {
hConf = HBASE_TEST_BASE.getConfiguration();
hConf.set(HBaseTableUtil.CFG_HBASE_TABLE_COMPRESSION, HBaseTableUtil.CompressionType.NONE.name());
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
cConf.set(Constants.CFG_HDFS_NAMESPACE, cConf.get(Constants.CFG_LOCAL_DATA_DIR));
cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
// Reduce the metadata cache refresh frequency for unit tests
cConf.set(Constants.MessagingSystem.COPROCESSOR_METADATA_CACHE_UPDATE_FREQUENCY_SECONDS, Integer.toString(METADATA_CACHE_EXPIRY));
hBaseAdmin = HBASE_TEST_BASE.getHBaseAdmin();
hBaseAdmin.getConfiguration().set(HBaseTableUtil.CFG_HBASE_TABLE_COMPRESSION, HBaseTableUtil.CompressionType.NONE.name());
tableUtil = new HBaseTableUtilFactory(cConf).get();
ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get();
ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
LocationFactory locationFactory = getInjector().getInstance(LocationFactory.class);
tableFactory = new HBaseTableFactory(cConf, hBaseAdmin.getConfiguration(), tableUtil, locationFactory);
new ConfigurationWriter(hConf, cConf).write(ConfigurationReader.Type.DEFAULT, cConf);
// write an initial transaction snapshot
invalidList.addAll(ImmutableList.of(V[3], V[5], V[7]));
TransactionSnapshot txSnapshot = TransactionSnapshot.copyFrom(System.currentTimeMillis(), V[6] - 1, V[7], invalidList, // this will set visibility upper bound to V[6]
Maps.newTreeMap(ImmutableSortedMap.of(V[6], new TransactionManager.InProgressTx(V[6] - 1, Long.MAX_VALUE, TransactionManager.InProgressType.SHORT))), new HashMap<Long, TransactionManager.ChangeSet>(), new TreeMap<Long, TransactionManager.ChangeSet>());
HDFSTransactionStateStorage tmpStorage = new HDFSTransactionStateStorage(hConf, new SnapshotCodecProvider(hConf), new TxMetricsCollector());
tmpStorage.startAndWait();
tmpStorage.writeSnapshot(txSnapshot);
tmpStorage.stopAndWait();
}
use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class DatasetAdminService method createOrUpdate.
/**
* Configures and creates a Dataset
*
* @param datasetInstanceId dataset instance to be created
* @param typeMeta type meta for the dataset
* @param props dataset instance properties
* @param existing if dataset already exists (in case of update), the existing properties
* @return dataset specification
*/
public DatasetSpecification createOrUpdate(final DatasetId datasetInstanceId, final DatasetTypeMeta typeMeta, final DatasetProperties props, @Nullable final DatasetSpecification existing) throws Exception {
if (existing == null) {
LOG.info("Creating dataset instance {}, type meta: {}", datasetInstanceId, typeMeta);
} else {
LOG.info("Updating dataset instance {}, type meta: {}, existing: {}", datasetInstanceId, typeMeta, existing);
}
try (DatasetClassLoaderProvider classLoaderProvider = new DirectoryClassLoaderProvider(cConf, locationFactory)) {
final DatasetContext context = DatasetContext.from(datasetInstanceId.getNamespace());
UserGroupInformation ugi = getUgiForDataset(impersonator, datasetInstanceId);
final DatasetType type = ImpersonationUtils.doAs(ugi, () -> {
DatasetType type1 = dsFramework.getDatasetType(typeMeta, null, classLoaderProvider);
if (type1 == null) {
throw new BadRequestException(String.format("Cannot instantiate dataset type using provided type meta: %s", typeMeta));
}
return type1;
});
DatasetSpecification spec = ImpersonationUtils.doAs(ugi, () -> {
DatasetSpecification spec1 = existing == null ? type.configure(datasetInstanceId.getEntityName(), props) : type.reconfigure(datasetInstanceId.getEntityName(), props, existing);
DatasetAdmin admin = type.getAdmin(context, spec1);
try {
if (existing != null) {
if (admin instanceof Updatable) {
((Updatable) admin).update(existing);
} else {
admin.upgrade();
}
} else {
admin.create();
}
} finally {
Closeables.closeQuietly(admin);
}
return spec1;
});
// Writing system metadata should be done without impersonation since user may not have access to system tables.
writeSystemMetadata(datasetInstanceId, spec, props, typeMeta, type, context, existing != null, ugi);
return spec;
} catch (Exception e) {
if (e instanceof IncompatibleUpdateException) {
// this is expected to happen if user provides bad update properties, so we log this as debug
LOG.debug("Incompatible update for dataset '{}'", datasetInstanceId, e);
} else {
LOG.error("Error {} dataset '{}': {}", existing == null ? "creating" : "updating", datasetInstanceId, e.getMessage(), e);
}
throw e;
}
}
Aggregations