use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.
the class FactScanner method createIterator.
private Iterator<FactScanResult> createIterator() {
return new AbstractIterator<FactScanResult>() {
@Override
protected FactScanResult computeNext() {
Row rowResult;
while ((rowResult = scanner.next()) != null) {
rowScanned++;
byte[] rowKey = rowResult.getRow();
// Decode context and metric from key
String measureName = codec.getMeasureName(rowKey);
// if measureNames is empty we include all metrics
if (!measureNames.isEmpty() && !measureNames.contains(measureName)) {
continue;
}
// todo: codec.getDimensionValues(rowKey) needs to un-encode dimension names which may result in read in
// entity table (depending on the cache and its state). To avoid that, we can pass to scanner the
// list of dimension names as we *always* know it (it is given) at the time of scanning
List<DimensionValue> dimensionValues = codec.getDimensionValues(rowKey);
boolean exhausted = false;
List<TimeValue> timeValues = Lists.newLinkedList();
// todo: entry set is ordered by ts?
for (Map.Entry<byte[], byte[]> columnValue : rowResult.getColumns().entrySet()) {
long ts = codec.getTimestamp(rowKey, columnValue.getKey());
if (ts < startTs) {
continue;
}
if (ts > endTs) {
exhausted = true;
break;
}
// todo: move Bytes.toLong into codec?
TimeValue timeValue = new TimeValue(ts, Bytes.toLong(columnValue.getValue()));
timeValues.add(timeValue);
}
if (timeValues.isEmpty() && exhausted) {
break;
}
// todo: can return empty list, if all data is < startTs or > endTs
return new FactScanResult(measureName, dimensionValues, timeValues);
}
scanner.close();
return endOfData();
}
};
}
use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.
the class FactTable method findSingleDimensionValue.
/**
* Searches for first non-null valued dimensions in records that contain given list of dimensions and match given
* dimension values in given time range. Returned dimension values are those that are not defined in given
* dimension values.
* @param allDimensionNames list of all dimension names to be present in the record
* @param dimensionSlice dimension values to filter by, {@code null} means any non-null value.
* @param startTs start of the time range, in seconds
* @param endTs end of the time range, in seconds
* @return {@link Set} of {@link DimensionValue}s
*/
// todo: pass a limit on number of dimensionValues returned
// todo: kinda not cool API when we expect null values in a map...
public Set<DimensionValue> findSingleDimensionValue(List<String> allDimensionNames, Map<String, String> dimensionSlice, long startTs, long endTs) {
// Algorithm, briefly:
// We scan in the records which have given allDimensionNames. We use dimensionSlice as a criteria for scan.
// If record from the scan has non-null values in the dimensions which are not specified in dimensionSlice,
// we use first of such dimension as a value to return.
// When we find value to return, since we only fill a single dimension, we are not interested in drilling down
// further and instead attempt to fast-forward (jump) to a record that has different value in that dimension.
// Thus we find all results.
List<DimensionValue> allDimensions = Lists.newArrayList();
List<Integer> dimToFillIndexes = Lists.newArrayList();
for (int i = 0; i < allDimensionNames.size(); i++) {
String dimensionName = allDimensionNames.get(i);
if (!dimensionSlice.containsKey(dimensionName)) {
dimToFillIndexes.add(i);
allDimensions.add(new DimensionValue(dimensionName, null));
} else {
DimensionValue dimensionValue = new DimensionValue(dimensionName, dimensionSlice.get(dimensionName));
allDimensions.add(dimensionValue);
}
}
// If provided dimensions contain all values filled in, there's nothing to look for
if (dimToFillIndexes.isEmpty()) {
return Collections.emptySet();
}
Set<DimensionValue> result = Sets.newHashSet();
int scans = 0;
int scannedRecords = 0;
// build a scan
byte[] startRow = codec.createStartRowKey(allDimensions, null, startTs, false);
byte[] endRow = codec.createEndRowKey(allDimensions, null, endTs, false);
endRow = Bytes.stopKeyForPrefix(endRow);
FuzzyRowFilter fuzzyRowFilter = createFuzzyRowFilter(new FactScan(startTs, endTs, Collections.emptyList(), allDimensions), startRow);
Scanner scanner = timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter);
scans++;
try {
Row rowResult;
while ((rowResult = scanner.next()) != null) {
scannedRecords++;
// todo: make configurable
if (scannedRecords > MAX_RECORDS_TO_SCAN_DURING_SEARCH) {
break;
}
byte[] rowKey = rowResult.getRow();
// filter out columns by time range (scan configuration only filters whole rows)
if (codec.getTimestamp(rowKey, codec.createColumn(startTs)) < startTs) {
continue;
}
if (codec.getTimestamp(rowKey, codec.createColumn(endTs)) > endTs) {
// we're done with scanner
break;
}
List<DimensionValue> dimensionValues = codec.getDimensionValues(rowResult.getRow());
// At this point, we know that the record is in right time range and its dimensions matches given.
// We try find first non-null valued dimension in the record that was not in given dimensions: we use it to form
// next drill down suggestion
int filledIndex = -1;
for (int index : dimToFillIndexes) {
// todo: it may be not efficient, if dimensionValues is not array-backed list: i.e. if access by index is
// not fast
DimensionValue dimensionValue = dimensionValues.get(index);
if (dimensionValue.getValue() != null) {
result.add(dimensionValue);
filledIndex = index;
break;
}
}
// todo: fast-forwarding (jumping) should be done on server-side (CDAP-1421)
if (filledIndex >= 0) {
scanner.close();
scanner = null;
scans++;
if (scans > MAX_SCANS_DURING_SEARCH) {
break;
}
startRow = codec.getNextRowKey(rowResult.getRow(), filledIndex);
scanner = timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter);
}
}
} finally {
if (scanner != null) {
scanner.close();
}
}
LOG.trace("search for dimensions completed, scans performed: {}, scanned records: {}", scans, scannedRecords);
return result;
}
use of io.cdap.cdap.api.dataset.table.Row in project cdap by cdapio.
the class TableAssert method assertScan.
public static void assertScan(byte[][] expectedRows, byte[][][] expectedRowMaps, Scanner scanner) {
for (int i = 0; i < expectedRows.length; i++) {
Row next = scanner.next();
Assert.assertNotNull(next);
Assert.assertArrayEquals(expectedRows[i], next.getRow());
assertRow(expectedRowMaps[i], next.getColumns());
}
// nothing is left in scan
Assert.assertNull(scanner.next());
}
use of io.cdap.cdap.api.dataset.table.Row in project hydrator-plugins by cdapio.
the class GroupByTestRun method testGroupBy.
private void testGroupBy(Engine engine) throws Exception {
/*
|--> group by user, totalPurchases:count(*), totalSpent:sum(price) --> user table
<ts, user, item, price> --|
|--> group by item, totalPurchases:count(user), latestPurchase:max(ts) --> item table
|
| test same name can be used in the output schema
|--> group by user, price:max(price) --> max table
*/
String purchasesDatasetName = "purchases-groupbytest-" + engine;
String usersDatasetName = "users-groupbytest-" + engine;
String itemsDatasetName = "items-groupbytest-" + engine;
String maxDatasetName = "max-groupbytest-" + engine;
Schema purchaseSchema = Schema.recordOf("purchase", Schema.Field.of("ts", Schema.of(Schema.Type.LONG)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("item", Schema.of(Schema.Type.STRING)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)));
ETLStage purchaseStage = new ETLStage("purchases", new ETLPlugin("Table", BatchSource.PLUGIN_TYPE, ImmutableMap.of(Properties.BatchReadableWritable.NAME, purchasesDatasetName, Properties.Table.PROPERTY_SCHEMA, purchaseSchema.toString()), null));
Schema userSchema = Schema.recordOf("user", Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("totalPurchases", Schema.of(Schema.Type.LONG)), Schema.Field.of("totalSpent", Schema.of(Schema.Type.DOUBLE)));
ETLStage userSinkStage = new ETLStage("users", new ETLPlugin("Table", BatchSink.PLUGIN_TYPE, ImmutableMap.of(Properties.BatchReadableWritable.NAME, usersDatasetName, Properties.Table.PROPERTY_SCHEMA_ROW_FIELD, "user", Properties.Table.PROPERTY_SCHEMA, userSchema.toString()), null));
Schema itemSchema = Schema.recordOf("item", Schema.Field.of("item", Schema.of(Schema.Type.STRING)), Schema.Field.of("totalPurchases", Schema.of(Schema.Type.LONG)), Schema.Field.of("latestPurchase", Schema.of(Schema.Type.LONG)));
ETLStage itemSinkStage = new ETLStage("items", new ETLPlugin("Table", BatchSink.PLUGIN_TYPE, ImmutableMap.of(Properties.BatchReadableWritable.NAME, itemsDatasetName, Properties.Table.PROPERTY_SCHEMA_ROW_FIELD, "item", Properties.Table.PROPERTY_SCHEMA, itemSchema.toString()), null));
ETLStage userGroupStage = new ETLStage("userGroup", new ETLPlugin("GroupByAggregate", BatchAggregator.PLUGIN_TYPE, ImmutableMap.of("groupByFields", "user", "aggregates", "totalPurchases:count(*), totalSpent:sum(price)"), null));
ETLStage itemGroupStage = new ETLStage("itemGroup", new ETLPlugin("GroupByAggregate", BatchAggregator.PLUGIN_TYPE, ImmutableMap.of("groupByFields", "item", "aggregates", "totalPurchases:count(user), latestPurchase:max(ts)"), null));
Schema maxSchema = Schema.recordOf("max", Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)));
ETLStage maxSinkStage = new ETLStage("max", new ETLPlugin("Table", BatchSink.PLUGIN_TYPE, ImmutableMap.of(Properties.BatchReadableWritable.NAME, maxDatasetName, Properties.Table.PROPERTY_SCHEMA_ROW_FIELD, "user", Properties.Table.PROPERTY_SCHEMA, maxSchema.toString()), null));
ETLStage maxGroupStage = new ETLStage("maxGroup", new ETLPlugin("GroupByAggregate", BatchAggregator.PLUGIN_TYPE, ImmutableMap.of("groupByFields", "user", "aggregates", "price:max(price)"), null));
ETLBatchConfig config = ETLBatchConfig.builder("* * * * *").addStage(purchaseStage).addStage(userSinkStage).addStage(itemSinkStage).addStage(maxSinkStage).addStage(userGroupStage).addStage(itemGroupStage).addStage(maxGroupStage).addConnection(purchaseStage.getName(), userGroupStage.getName()).addConnection(purchaseStage.getName(), itemGroupStage.getName()).addConnection(purchaseStage.getName(), maxGroupStage.getName()).addConnection(userGroupStage.getName(), userSinkStage.getName()).addConnection(itemGroupStage.getName(), itemSinkStage.getName()).addConnection(maxGroupStage.getName(), maxSinkStage.getName()).setEngine(engine).build();
ApplicationManager appManager = deployETL(config, "groupby-test-" + engine);
// write input data
// 1: 1234567890000, samuel, island, 1000000
// 2: 1234567890001, samuel, shirt, 15.34
// 3. 1234567890001, samuel, pie, 3.14
// 4. 1234567890002, john, pie, 3.14
// 5. 1234567890003, john, shirt, 20.53
DataSetManager<Table> purchaseManager = getDataset(purchasesDatasetName);
Table purchaseTable = purchaseManager.get();
// 1: 1234567890000, samuel, island, 1000000
Put put = new Put(Bytes.toBytes(1));
put.add("ts", 1234567890000L);
put.add("user", "samuel");
put.add("item", "island");
put.add("price", 1000000d);
purchaseTable.put(put);
put = new Put(Bytes.toBytes(2));
put.add("ts", 1234567890001L);
put.add("user", "samuel");
put.add("item", "shirt");
put.add("price", 15.34d);
purchaseTable.put(put);
put = new Put(Bytes.toBytes(3));
put.add("ts", 1234567890001L);
put.add("user", "samuel");
put.add("item", "pie");
put.add("price", 3.14d);
purchaseTable.put(put);
put = new Put(Bytes.toBytes(4));
put.add("ts", 1234567890002L);
put.add("user", "john");
put.add("item", "pie");
put.add("price", 3.14d);
purchaseTable.put(put);
put = new Put(Bytes.toBytes(5));
put.add("ts", 1234567890003L);
put.add("user", "john");
put.add("item", "shirt");
put.add("price", 20.53d);
purchaseTable.put(put);
purchaseManager.flush();
// run the pipeline
runETLOnce(appManager);
DataSetManager<Table> usersManager = getDataset(usersDatasetName);
Table usersTable = usersManager.get();
// users table should have:
// samuel: 3, 1000000 + 15.34 + 3.14
Row row = usersTable.get(Bytes.toBytes("samuel"));
Assert.assertEquals(Objects.requireNonNull(row.getLong("totalPurchases")).longValue(), 3L);
Assert.assertTrue(Math.abs(Objects.requireNonNull(row.getDouble("totalSpent")) - 1000000d - 15.34d - 3.14d) < 0.0000001);
// john: 2, 3.14 + 20.53
row = usersTable.get(Bytes.toBytes("john"));
Assert.assertEquals(Objects.requireNonNull(row.getLong("totalPurchases")).longValue(), 2L);
Assert.assertTrue(Math.abs(Objects.requireNonNull(row.getDouble("totalSpent")) - 3.14d - 20.53d) < 0.0000001);
DataSetManager<Table> itemsManager = getDataset(itemsDatasetName);
Table itemsTable = itemsManager.get();
// items table should have:
// island: 1, 1234567890000
row = itemsTable.get(Bytes.toBytes("island"));
Assert.assertEquals(Objects.requireNonNull(row.getLong("totalPurchases")).longValue(), 1L);
Assert.assertEquals(Objects.requireNonNull(row.getLong("latestPurchase")).longValue(), 1234567890000L);
// pie: 2, 1234567890002
row = itemsTable.get(Bytes.toBytes("pie"));
Assert.assertEquals(Objects.requireNonNull(row.getLong("totalPurchases")).longValue(), 2L);
Assert.assertEquals(Objects.requireNonNull(row.getLong("latestPurchase")).longValue(), 1234567890002L);
// shirt: 2, 1234567890003
row = itemsTable.get(Bytes.toBytes("shirt"));
Assert.assertEquals(Objects.requireNonNull(row.getLong("totalPurchases")).longValue(), 2L);
Assert.assertEquals(Objects.requireNonNull(row.getLong("latestPurchase")).longValue(), 1234567890003L);
DataSetManager<Table> maxManager = getDataset(maxDatasetName);
Table maxTable = maxManager.get();
// max table should have:
// Samuel, 100000
row = maxTable.get(Bytes.toBytes("samuel"));
Assert.assertTrue(Math.abs(row.getDouble("price") - 1000000d) < 0.0000001);
// pie: 2, 1234567890002
row = maxTable.get(Bytes.toBytes("john"));
Assert.assertTrue(Math.abs(row.getDouble("price") - 20.53d) < 0.0000001);
}
use of io.cdap.cdap.api.dataset.table.Row in project hydrator-plugins by cdapio.
the class ETLMapReduceTestRun method testTableToTableWithValidations.
@SuppressWarnings("ConstantConditions")
@Test
public void testTableToTableWithValidations() throws Exception {
Schema schema = Schema.recordOf("purchase", Schema.Field.of("rowkey", Schema.of(Schema.Type.STRING)), Schema.Field.of("user", Schema.of(Schema.Type.STRING)), Schema.Field.of("count", Schema.of(Schema.Type.INT)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE)), Schema.Field.of("item", Schema.of(Schema.Type.STRING)));
ETLStage source = new ETLStage("source", new ETLPlugin("Table", BatchSource.PLUGIN_TYPE, ImmutableMap.of(Properties.BatchReadableWritable.NAME, "inputTable", Properties.Table.PROPERTY_SCHEMA_ROW_FIELD, "rowkey", Properties.Table.PROPERTY_SCHEMA, schema.toString()), null));
ETLStage sink1 = new ETLStage("sink1", new ETLPlugin("Table", BatchSink.PLUGIN_TYPE, ImmutableMap.of(Properties.BatchReadableWritable.NAME, "outputTable", Properties.Table.PROPERTY_SCHEMA_ROW_FIELD, "rowkey", Properties.Table.PROPERTY_SCHEMA, schema.toString())));
ETLBatchConfig etlConfig = ETLBatchConfig.builder("* * * * *").addStage(source).addStage(sink1).addConnection(source.getName(), sink1.getName()).build();
ApplicationManager appManager = deployETL(etlConfig, "TableToTable");
// add some data to the input table
DataSetManager<Table> inputManager = getDataset("inputTable");
Table inputTable = inputManager.get();
// valid record, user name "samuel" is 6 chars long
Put put = new Put(Bytes.toBytes("row1"));
put.add("user", "samuel");
put.add("count", 5);
put.add("price", 123.45);
put.add("item", "scotch");
inputTable.put(put);
inputManager.flush();
// valid record, user name "jackson" is > 6 characters
put = new Put(Bytes.toBytes("row2"));
put.add("user", "jackson");
put.add("count", 10);
put.add("price", 123456789d);
put.add("item", "island");
inputTable.put(put);
inputManager.flush();
runETLOnce(appManager);
DataSetManager<Table> outputManager = getDataset("outputTable");
Table outputTable = outputManager.get();
Row row = outputTable.get(Bytes.toBytes("row1"));
Assert.assertEquals("samuel", row.getString("user"));
Assert.assertEquals(5, (int) row.getInt("count"));
Assert.assertTrue(Math.abs(123.45 - row.getDouble("price")) < 0.000001);
Assert.assertEquals("scotch", row.getString("item"));
}
Aggregations