Examples with Get co.cask.cdap.api.dataset.table.Get used on opensource projects

Example 11 with Get

use of co.cask.cdap.api.dataset.table.Get in project cdap by caskdata.

the class BufferingTable method getPersisted.

/**
   * Fetches a list of rows from persistent store. Subclasses should override this if they can batch multiple
   * gets into a single request, as the default implementation simply loops through the gets and calls
   * {@link #getPersisted(byte[], byte[][])} on each get.
   * NOTE: persisted store can also be in-memory, it is called "persisted" to distinguish from in-memory buffer.
   * @param gets list of gets to perform
   * @return list of rows, one for each get
   * @throws Exception
   */
protected List<Map<byte[], byte[]>> getPersisted(List<Get> gets) throws Exception {
    List<Map<byte[], byte[]>> results = Lists.newArrayListWithCapacity(gets.size());
    for (Get get : gets) {
        List<byte[]> getColumns = get.getColumns();
        byte[][] columns = getColumns == null ? null : getColumns.toArray(new byte[getColumns.size()][]);
        results.add(getPersisted(get.getRow(), columns));
    }
    return results;
}

Example 12 with Get

use of co.cask.cdap.api.dataset.table.Get in project cdap by caskdata.

the class BufferingTable method get.

@ReadOnly
@Override
public List<Row> get(List<Get> gets) {
    ensureTransactionIsStarted();
    try {
        // get persisted, then overwrite with whats buffered
        List<Map<byte[], byte[]>> persistedRows = getPersisted(gets);
        // gets and rows lists are always of the same size
        Preconditions.checkArgument(gets.size() == persistedRows.size(), "Invalid number of rows fetched when performing multi-get. There must be one row for each get.");
        List<Row> result = Lists.newArrayListWithCapacity(persistedRows.size());
        Iterator<Map<byte[], byte[]>> persistedRowsIter = persistedRows.iterator();
        Iterator<Get> getIter = gets.iterator();
        while (persistedRowsIter.hasNext() && getIter.hasNext()) {
            Get get = getIter.next();
            Map<byte[], byte[]> persistedRow = persistedRowsIter.next();
            // navigable copy of the persisted data. Implementation may return immutable or unmodifiable maps,
            // so we make a copy here.
            NavigableMap<byte[], byte[]> rowColumns = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
            rowColumns.putAll(persistedRow);
            byte[] row = get.getRow();
            NavigableMap<byte[], Update> buffCols = buff.get(row);
            // merge what was in the buffer and what was persisted
            if (buffCols != null) {
                List<byte[]> getColumns = get.getColumns();
                byte[][] columns = getColumns == null ? null : getColumns.toArray(new byte[getColumns.size()][]);
                mergeToPersisted(rowColumns, buffCols, columns);
            }
            result.add(new Result(row, unwrapDeletes(rowColumns)));
        }
        return result;
    } catch (Exception e) {
        LOG.debug("multi-get failed for table: " + getTransactionAwareName(), e);
        throw new DataSetException("multi-get failed", e);
    }
}

Example 13 with Get

use of co.cask.cdap.api.dataset.table.Get in project cdap by caskdata.

the class DatasetOpExecutorServiceTest method testRest.

@Test
public void testRest() throws Exception {
    // check non-existence with 404
    testAdminOp(bob, "exists", 404, null);
    // add instance, should automatically create an instance
    dsFramework.addInstance("table", bob, DatasetProperties.EMPTY);
    testAdminOp(bob, "exists", 200, true);
    testAdminOp("bob", "exists", 404, null);
    // check truncate
    final Table table = dsFramework.getDataset(bob, DatasetDefinition.NO_ARGUMENTS, null);
    Assert.assertNotNull(table);
    TransactionExecutor txExecutor = new DefaultTransactionExecutor(new InMemoryTxSystemClient(txManager), ImmutableList.of((TransactionAware) table));
    // writing smth to table
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            table.put(new Put("key1", "col1", "val1"));
        }
    });
    // verify that we can read the data
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals("val1", table.get(new Get("key1", "col1")).getString("col1"));
        }
    });
    testAdminOp(bob, "truncate", 200, null);
    // verify that data is no longer there
    txExecutor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertTrue(table.get(new Get("key1", "col1")).isEmpty());
        }
    });
    // check upgrade
    testAdminOp(bob, "upgrade", 200, null);
    // drop and check non-existence
    dsFramework.deleteInstance(bob);
    testAdminOp(bob, "exists", 404, null);
}

Example 14 with Get

use of co.cask.cdap.api.dataset.table.Get in project cdap by caskdata.

the class RetrieveCountsHandler method getStats.

/**
   * Returns total number of words, the number of unique words, and the average word length.
   */
@Path("stats")
@GET
public void getStats(HttpServiceRequest request, HttpServiceResponder responder) {
    long totalWords = 0L;
    long uniqueWords = 0L;
    double averageLength = 0.0;
    // Read the total_length and total_words to calculate average length
    Row result = wordStatsTable.get(new Get("totals", "total_length", "total_words"));
    if (!result.isEmpty()) {
        // Extract the total sum of lengths
        long totalLength = result.getLong("total_length", 0);
        // Extract the total count of words
        totalWords = result.getLong("total_words", 0);
        // Compute the average length
        if (totalLength != 0 && totalWords != 0) {
            averageLength = ((double) totalLength) / totalWords;
            // Read the unique word count
            uniqueWords = uniqueCountTable.readUniqueCount();
        }
    }
    // Return a map as JSON
    Map<String, Object> results = new HashMap<>();
    results.put("totalWords", totalWords);
    results.put("uniqueWords", uniqueWords);
    results.put("averageLength", averageLength);
    responder.sendJson(results);
}

Example 15 with Get

use of co.cask.cdap.api.dataset.table.Get in project cdap by caskdata.

the class UserProfilesTest method testUserProfiles.

@Test
public void testUserProfiles() throws Exception {
    // deploy the app
    ApplicationManager applicationManager = deployApplication(UserProfiles.class);
    // run the service and the flow
    FlowManager flowManager = applicationManager.getFlowManager("ActivityFlow").start();
    ServiceManager serviceManager = applicationManager.getServiceManager("UserProfileService").start();
    serviceManager.waitForStatus(true);
    URL serviceURL = serviceManager.getServiceURL();
    // create a user through the service
    String userJson = new Gson().toJson(ImmutableMap.of("id", "1234", "name", "joe", "email", "joe@bla.ck"));
    HttpURLConnection connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(userJson.getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_CREATED, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // read the user through the dataset
    DataSetManager<Table> tableManager = getDataset("profiles");
    Row row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@bla.ck", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // update email address through service
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/email").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write("joe@black.com".getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the updated email address
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertNull(row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send a login event
    long loginTime = System.currentTimeMillis();
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234/lastLogin").openConnection();
    try {
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");
        connection.getOutputStream().write(Long.toString(loginTime).getBytes(Charsets.UTF_8));
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the login time through the dataset
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertNull(row.getLong("active"));
    // send an event to the stream
    long activeTime = System.currentTimeMillis();
    StreamManager streamManager = getStreamManager("events");
    streamManager.send(new Gson().toJson(new Event(activeTime, "1234", "/some/path")));
    try {
        // Wait for the last Flowlet processing 1 events, or at most 5 seconds
        RuntimeMetrics metrics = flowManager.getFlowletMetrics("updater");
        metrics.waitForProcessed(1, 5, TimeUnit.SECONDS);
    } finally {
        flowManager.stop();
        Assert.assertFalse(flowManager.isRunning());
    }
    // verify the last active time for the user
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertEquals("1234", row.getString("id"));
    Assert.assertEquals("joe", row.getString("name"));
    Assert.assertEquals("joe@black.com", row.getString("email"));
    Assert.assertEquals(new Long(loginTime), row.getLong("login"));
    Assert.assertEquals(new Long(activeTime), row.getLong("active"));
    // delete the user
    connection = (HttpURLConnection) new URL(serviceURL, "profiles/1234").openConnection();
    try {
        connection.setRequestMethod("DELETE");
        Assert.assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
    } finally {
        connection.disconnect();
    }
    // verify the user is gone
    tableManager.flush();
    row = tableManager.get().get(new Get("1234"));
    Assert.assertTrue(row.isEmpty());
    // stop the service and the flow
    serviceManager.stop();
}