Examples with Row co.cask.cdap.api.dataset.table.Row used on opensource projects

Example 31 with Row

use of co.cask.cdap.api.dataset.table.Row in project cdap by caskdata.

the class TableAssert method assertScan.

public static void assertScan(byte[][] expectedRows, byte[][][] expectedRowMaps, Scanner scanner) {
    for (int i = 0; i < expectedRows.length; i++) {
        Row next = scanner.next();
        Assert.assertNotNull(next);
        Assert.assertArrayEquals(expectedRows[i], next.getRow());
        assertRow(expectedRowMaps[i], next.getColumns());
    }
    // nothing is left in scan
    Assert.assertNull(scanner.next());
}

Example 32 with Row

use of co.cask.cdap.api.dataset.table.Row in project cdap by caskdata.

the class RetrieveCountsHandler method getStats.

/**
   * Returns total number of words, the number of unique words, and the average word length.
   */
@Path("stats")
@GET
public void getStats(HttpServiceRequest request, HttpServiceResponder responder) {
    long totalWords = 0L;
    long uniqueWords = 0L;
    double averageLength = 0.0;
    // Read the total_length and total_words to calculate average length
    Row result = wordStatsTable.get(new Get("totals", "total_length", "total_words"));
    if (!result.isEmpty()) {
        // Extract the total sum of lengths
        long totalLength = result.getLong("total_length", 0);
        // Extract the total count of words
        totalWords = result.getLong("total_words", 0);
        // Compute the average length
        if (totalLength != 0 && totalWords != 0) {
            averageLength = ((double) totalLength) / totalWords;
            // Read the unique word count
            uniqueWords = uniqueCountTable.readUniqueCount();
        }
    }
    // Return a map as JSON
    Map<String, Object> results = new HashMap<>();
    results.put("totalWords", totalWords);
    results.put("uniqueWords", uniqueWords);
    results.put("averageLength", averageLength);
    responder.sendJson(results);
}

Example 33 with Row

use of co.cask.cdap.api.dataset.table.Row in project cdap by caskdata.

the class UserProfilesTest method testUserProfiles.

@Test
public void testUserProfiles() throws Exception {
    // deploy the app
    ApplicationManager applicationManager = deployApplication(UserProfiles.class);
    // run the service and the flow
    FlowManager flowManager = applicationManager.getFlowManager("ActivityFlow").start();
    ServiceManager serviceManager = applicationManager.getServiceManager("UserProfileService").start();
    serviceManager.waitForStatus(true);
    URL serviceURL = serviceManager.getServiceURL();
    // create a user through the service
    String userJson = new Gson().toJson(ImmutableMap.of("id", "1234", "name", "joe", "email", "joe@bla.ck"));
    HttpURLConnection connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(userJson.getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_CREATED, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // read the user through the dataset
    DataSetManager<Table> tableManager = getDataset("profiles");
    Row row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@bla.ck", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // update email address through service
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/email").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write("joe@black.com".getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the updated email address
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send a login event
    long loginTime = System.currentTimeMillis();
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/lastLogin").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(Long.toString(loginTime).getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the login time through the dataset
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send an event to the stream
    long activeTime = System.currentTimeMillis();
    StreamManager streamManager = getStreamManager("events");
    streamManager.send(new Gson().toJson(new Event(activeTime, "1234", "/some/path")));
    try {
        // Wait for the last Flowlet processing 1 events, or at most 5 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("updater");
        metrics.waitForProcessed(1, 5, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
        Assert.assertFalse(flowManager.isRunning());
    }
    // verify the last active time for the user
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertEquals(new Long(activeTime), row.getLong("active"));
    // delete the user
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setRequestMethod("DELETE");
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the user is gone
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertTrue(row.isEmpty());
    // stop the service and the flow
    serviceManager.stop();
}

Example 34 with Row

use of co.cask.cdap.api.dataset.table.Row in project cdap by caskdata.

the class FileMetadataCleaner method scanAndGetFilesToDelete.

/**
   * scans for meta data in new format which has expired the log retention.
   * @param tillTime time till which files will be deleted
   * @param transactionTimeout transaction timeout to use for scanning entries, deleting entries.
   * @return list of DeleteEntry - used to get files to delete for which metadata has already been deleted
   */
public List<DeletedEntry> scanAndGetFilesToDelete(final long tillTime, final int transactionTimeout) {
    final List<DeletedEntry> toDelete = new ArrayList<>();
    // we make sure transactionTimeout is greater than TX_TIMEOUT_DISCOUNT_SECS in CDAPLogAppender check.
    final int cutOffTransactionTime = transactionTimeout - TX_TIMEOUT_DISCOUNT_SECS;
    try {
        transactional.execute(transactionTimeout, new TxRunnable() {

            @Override
            public void run(DatasetContext context) throws Exception {
                Table table = LoggingStoreTableUtil.getMetadataTable(context, datasetManager);
                Stopwatch stopwatch = new Stopwatch().start();
                byte[] startRowKey = NEW_ROW_KEY_PREFIX;
                byte[] endRowKey = NEW_ROW_KEY_PREFIX_END;
                boolean reachedEnd = false;
                while (!reachedEnd) {
                    try (Scanner scanner = table.scan(startRowKey, endRowKey)) {
                        while (stopwatch.elapsedTime(TimeUnit.SECONDS) < cutOffTransactionTime) {
                            Row row = scanner.next();
                            if (row == null) {
                                // if row is null, then scanner next returned null. so we have reached the end.
                                reachedEnd = true;
                                break;
                            }
                            byte[] rowkey = row.getRow();
                            // file creation time is the last 8-bytes in rowkey in the new format
                            long creationTime = Bytes.toLong(rowkey, rowkey.length - Bytes.SIZEOF_LONG, Bytes.SIZEOF_LONG);
                            if (creationTime <= tillTime) {
                                // expired - can be deleted
                                toDelete.add(new DeletedEntry(rowkey, Bytes.toString(row.get(LoggingStoreTableUtil.META_TABLE_COLUMN_KEY))));
                            } else {
                                // update start-row key based on the logging context and start a new scan.
                                startRowKey = Bytes.add(NEW_ROW_KEY_PREFIX, getNextContextStartKey(rowkey));
                                break;
                            }
                        }
                    }
                }
            }
        });
    } catch (TransactionFailureException e) {
        LOG.warn("Got Exception while scanning metadata table", e);
        // if there is an exception, no metadata, so delete file should be skipped.
        return new ArrayList<>();
    }
    if (!toDelete.isEmpty()) {
        // we will call delete on old metadata even whenever there is expired entries to delete in new format.
        // though the first call will delete all old meta data.
        scanAndDeleteOldMetaData(transactionTimeout, cutOffTransactionTime);
        // delete meta data entries in toDelete and get the file location list
        return deleteNewMetadataEntries(toDelete, transactionTimeout, cutOffTransactionTime);
    }
    // toDelete is empty, safe to return that
    return toDelete;
}

Example 35 with Row

use of co.cask.cdap.api.dataset.table.Row in project cdap by caskdata.

the class DatasetBasedStreamSizeScheduleStoreTest method testDeletion.

private void testDeletion(final ProgramId programId) throws Exception {
    final boolean defaultVersion = programId.getVersion().equals(ApplicationId.DEFAULT_VERSION);
    DatasetId storeTable = NamespaceId.SYSTEM.dataset(ScheduleStoreTableUtil.SCHEDULE_STORE_DATASET_NAME);
    final Table table = datasetFramework.getDataset(storeTable, ImmutableMap.<String, String>of(), null);
    Assert.assertNotNull(table);
    TransactionExecutor txnl = txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) table));
    final byte[] startKey = Bytes.toBytes(DatasetBasedStreamSizeScheduleStore.KEY_PREFIX);
    final byte[] stopKey = Bytes.stopKeyForPrefix(startKey);
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Scanner scanner = table.scan(startKey, stopKey);
            Assert.assertNull(scanner.next());
            scanner.close();
        }
    });
    // Create one stream schedule - this will be persisted with new format
    scheduleStore.persist(programId, PROGRAM_TYPE, STREAM_SCHEDULE_1, MAP_1, 0L, 0L, 0L, 0L, true);
    // Create one stream schedule - based on the old format
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            // Create a programId without version so that we can create a old format schedule
            ProgramId defaultProgramId = new ProgramId(programId.getNamespace(), programId.getApplication(), programId.getType(), programId.getProgram());
            String newRowKey = scheduleStore.getRowKey(defaultProgramId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName());
            Row row = table.get(Bytes.toBytes(scheduleStore.getRowKey(programId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName())));
            Assert.assertFalse(row.isEmpty());
            byte[] oldRowKey = Bytes.toBytes(scheduleStore.removeAppVersion(newRowKey));
            for (Map.Entry<byte[], byte[]> entry : row.getColumns().entrySet()) {
                table.put(oldRowKey, entry.getKey(), entry.getValue());
            }
        }
    });
    // Make sure there are only two stream size schedules
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Scanner scanner = table.scan(startKey, stopKey);
            int numRows = 0;
            while (true) {
                Row row = scanner.next();
                if (row == null) {
                    break;
                }
                numRows++;
            }
            scanner.close();
            Assert.assertEquals(2, numRows);
        }
    });
    // This delete should have deleted both the old and new row format
    scheduleStore.delete(programId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName());
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Scanner scanner = table.scan(startKey, stopKey);
            if (defaultVersion) {
                Assert.assertNull(scanner.next());
            } else {
                Assert.assertNotNull(scanner.next());
                Assert.assertNull(scanner.next());
            }
            scanner.close();
        }
    });
    // If the version is not default, we need to delete the row which didn't have a version
    if (!defaultVersion) {
        txnl.execute(new TransactionExecutor.Subroutine() {

            @Override
            public void apply() throws Exception {
                // Create a programId without version so that we can create row key to delete the old format schedule
                ProgramId defaultProgramId = new ProgramId(programId.getNamespace(), programId.getApplication(), programId.getType(), programId.getProgram());
                String newRowKey = scheduleStore.getRowKey(defaultProgramId, PROGRAM_TYPE, STREAM_SCHEDULE_1.getName());
                byte[] oldRowKey = Bytes.toBytes(scheduleStore.removeAppVersion(newRowKey));
                Row row = table.get(oldRowKey);
                Assert.assertFalse(row.isEmpty());
                table.delete(oldRowKey);
            }
        });
    }
}