Examples with Result org.apache.hadoop.hbase.client.Result used on opensource projects

Example 46 with Result

use of org.apache.hadoop.hbase.client.Result in project hadoop by apache.

the class TestHBaseTimelineStorageEntities method testEventsWithEmptyInfo.

@Test
public void testEventsWithEmptyInfo() throws IOException {
    TimelineEvent event = new TimelineEvent();
    String eventId = "foo_ev e  nt_id";
    event.setId(eventId);
    Long expTs = 1436512802000L;
    event.setTimestamp(expTs);
    final TimelineEntity entity = new TimelineEntity();
    entity.setId("attempt_1329348432655_0001_m_000008_18");
    entity.setType("FOO_ATTEMPT");
    entity.addEvent(event);
    TimelineEntities entities = new TimelineEntities();
    entities.addEntity(entity);
    HBaseTimelineWriterImpl hbi = null;
    try {
        Configuration c1 = util.getConfiguration();
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.start();
        String cluster = "cluster_test_empty_eventkey";
        String user = "user_emptyeventkey";
        String flow = "other_flow_name";
        String flowVersion = "1111F01C2287BA";
        long runid = 1009876543218L;
        String appName = ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1).toString();
        byte[] startRow = new EntityRowKeyPrefix(cluster, user, flow, runid, appName).getRowKeyPrefix();
        hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
        hbi.stop();
        // scan the table and see that entity exists
        Scan s = new Scan();
        s.setStartRow(startRow);
        s.addFamily(EntityColumnFamily.INFO.getBytes());
        Connection conn = ConnectionFactory.createConnection(c1);
        ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
        int rowCount = 0;
        for (Result result : scanner) {
            if (result != null && !result.isEmpty()) {
                rowCount++;
                // check the row key
                byte[] row1 = result.getRow();
                assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName, entity));
                Map<EventColumnName, Object> eventsResult = EntityColumnPrefix.EVENT.readResults(result, new EventColumnNameConverter());
                // there should be only one event
                assertEquals(1, eventsResult.size());
                for (Map.Entry<EventColumnName, Object> e : eventsResult.entrySet()) {
                    EventColumnName eventColumnName = e.getKey();
                    // the qualifier is a compound key
                    // hence match individual values
                    assertEquals(eventId, eventColumnName.getId());
                    assertEquals(expTs, eventColumnName.getTimestamp());
                    // key must be empty
                    assertNull(eventColumnName.getInfoKey());
                    Object value = e.getValue();
                    // value should be empty
                    assertEquals("", value.toString());
                }
            }
        }
        assertEquals(1, rowCount);
        // read the timeline entity using the reader this time
        TimelineEntity e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        Set<TimelineEntity> es1 = reader.getEntities(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        assertNotNull(e1);
        assertEquals(1, es1.size());
        // check the events
        NavigableSet<TimelineEvent> events = e1.getEvents();
        // there should be only one event
        assertEquals(1, events.size());
        for (TimelineEvent e : events) {
            assertEquals(eventId, e.getId());
            assertEquals(expTs, Long.valueOf(e.getTimestamp()));
            Map<String, Object> info = e.getInfo();
            assertTrue(info == null || info.isEmpty());
        }
    } finally {
        if (hbi != null) {
            hbi.stop();
            hbi.close();
        }
    }
}

Example 47 with Result

use of org.apache.hadoop.hbase.client.Result in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testEvents.

@Test
public void testEvents() throws IOException {
    TimelineEvent event = new TimelineEvent();
    String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
    event.setId(eventId);
    Long expTs = 1436512802000L;
    event.setTimestamp(expTs);
    String expKey = "foo_event";
    Object expVal = "test";
    event.addInfo(expKey, expVal);
    final TimelineEntity entity = new ApplicationEntity();
    entity.setId(ApplicationId.newInstance(0, 1).toString());
    entity.addEvent(event);
    TimelineEntities entities = new TimelineEntities();
    entities.addEntity(entity);
    HBaseTimelineWriterImpl hbi = null;
    try {
        Configuration c1 = util.getConfiguration();
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.start();
        String cluster = "cluster_test_events";
        String user = "user2";
        String flow = "other_flow_name";
        String flowVersion = "1111F01C2287BA";
        long runid = 1009876543218L;
        String appName = "application_123465899910_1001";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
        hbi.stop();
        // retrieve the row
        ApplicationRowKey applicationRowKey = new ApplicationRowKey(cluster, user, flow, runid, appName);
        byte[] rowKey = applicationRowKey.getRowKey();
        Get get = new Get(rowKey);
        get.setMaxVersions(Integer.MAX_VALUE);
        Connection conn = ConnectionFactory.createConnection(c1);
        Result result = new ApplicationTable().getResult(c1, conn, get);
        assertTrue(result != null);
        // check the row key
        byte[] row1 = result.getRow();
        assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid, appName));
        Map<EventColumnName, Object> eventsResult = ApplicationColumnPrefix.EVENT.readResults(result, new EventColumnNameConverter());
        // there should be only one event
        assertEquals(1, eventsResult.size());
        for (Map.Entry<EventColumnName, Object> e : eventsResult.entrySet()) {
            EventColumnName eventColumnName = e.getKey();
            // the qualifier is a compound key
            // hence match individual values
            assertEquals(eventId, eventColumnName.getId());
            assertEquals(expTs, eventColumnName.getTimestamp());
            assertEquals(expKey, eventColumnName.getInfoKey());
            Object value = e.getValue();
            // there should be only one timestamp and value
            assertEquals(expVal, value.toString());
        }
        // read the timeline entity using the reader this time
        TimelineEntity e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        TimelineEntity e2 = reader.getEntity(new TimelineReaderContext(cluster, user, null, null, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        assertNotNull(e1);
        assertNotNull(e2);
        assertEquals(e1, e2);
        // check the events
        NavigableSet<TimelineEvent> events = e1.getEvents();
        // there should be only one event
        assertEquals(1, events.size());
        for (TimelineEvent e : events) {
            assertEquals(eventId, e.getId());
            assertEquals(expTs, Long.valueOf(e.getTimestamp()));
            Map<String, Object> info = e.getInfo();
            assertEquals(1, info.size());
            for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
                assertEquals(expKey, infoEntry.getKey());
                assertEquals(expVal, infoEntry.getValue());
            }
        }
    } finally {
        if (hbi != null) {
            hbi.stop();
            hbi.close();
        }
    }
}

Example 48 with Result

use of org.apache.hadoop.hbase.client.Result in project hadoop by apache.

the class TestHBaseTimelineStorageEntities method testWriteEntityToHBase.

@Test
public void testWriteEntityToHBase() throws Exception {
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    String id = "hello";
    String type = "world";
    entity.setId(id);
    entity.setType(type);
    Long cTime = 1425016501000L;
    entity.setCreatedTime(cTime);
    // add the info map in Timeline Entity
    Map<String, Object> infoMap = new HashMap<String, Object>();
    infoMap.put("infoMapKey1", "infoMapValue1");
    infoMap.put("infoMapKey2", 10);
    entity.addInfo(infoMap);
    // add the isRelatedToEntity info
    String key = "task";
    String value = "is_related_to_entity_id_here";
    Set<String> isRelatedToSet = new HashSet<String>();
    isRelatedToSet.add(value);
    Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
    isRelatedTo.put(key, isRelatedToSet);
    entity.setIsRelatedToEntities(isRelatedTo);
    // add the relatesTo info
    key = "container";
    value = "relates_to_entity_id_here";
    Set<String> relatesToSet = new HashSet<String>();
    relatesToSet.add(value);
    value = "relates_to_entity_id_here_Second";
    relatesToSet.add(value);
    Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
    relatesTo.put(key, relatesToSet);
    entity.setRelatesToEntities(relatesTo);
    // add some config entries
    Map<String, String> conf = new HashMap<String, String>();
    conf.put("config_param1", "value1");
    conf.put("config_param2", "value2");
    entity.addConfigs(conf);
    // add metrics
    Set<TimelineMetric> metrics = new HashSet<>();
    TimelineMetric m1 = new TimelineMetric();
    m1.setId("MAP_SLOT_MILLIS");
    Map<Long, Number> metricValues = new HashMap<Long, Number>();
    long ts = System.currentTimeMillis();
    metricValues.put(ts - 120000, 100000000);
    metricValues.put(ts - 100000, 200000000);
    metricValues.put(ts - 80000, 300000000);
    metricValues.put(ts - 60000, 400000000);
    metricValues.put(ts - 40000, 50000000000L);
    metricValues.put(ts - 20000, 60000000000L);
    m1.setType(Type.TIME_SERIES);
    m1.setValues(metricValues);
    metrics.add(m1);
    entity.addMetrics(metrics);
    te.addEntity(entity);
    HBaseTimelineWriterImpl hbi = null;
    try {
        Configuration c1 = util.getConfiguration();
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.start();
        String cluster = "cluster_test_write_entity";
        String user = "user1";
        String flow = "some_flow_name";
        String flowVersion = "AB7822C10F1111";
        long runid = 1002345678919L;
        String appName = ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1).toString();
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        hbi.stop();
        // scan the table and see that entity exists
        Scan s = new Scan();
        byte[] startRow = new EntityRowKeyPrefix(cluster, user, flow, runid, appName).getRowKeyPrefix();
        s.setStartRow(startRow);
        s.setMaxVersions(Integer.MAX_VALUE);
        Connection conn = ConnectionFactory.createConnection(c1);
        ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
        int rowCount = 0;
        int colCount = 0;
        KeyConverter<String> stringKeyConverter = new StringKeyConverter();
        for (Result result : scanner) {
            if (result != null && !result.isEmpty()) {
                rowCount++;
                colCount += result.size();
                byte[] row1 = result.getRow();
                assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName, entity));
                // check info column family
                String id1 = EntityColumn.ID.readResult(result).toString();
                assertEquals(id, id1);
                String type1 = EntityColumn.TYPE.readResult(result).toString();
                assertEquals(type, type1);
                Long cTime1 = (Long) EntityColumn.CREATED_TIME.readResult(result);
                assertEquals(cTime1, cTime);
                Map<String, Object> infoColumns = EntityColumnPrefix.INFO.readResults(result, new StringKeyConverter());
                assertEquals(infoMap, infoColumns);
                // Remember isRelatedTo is of type Map<String, Set<String>>
                for (Map.Entry<String, Set<String>> isRelatedToEntry : isRelatedTo.entrySet()) {
                    Object isRelatedToValue = EntityColumnPrefix.IS_RELATED_TO.readResult(result, isRelatedToEntry.getKey());
                    String compoundValue = isRelatedToValue.toString();
                    // id7?id9?id6
                    Set<String> isRelatedToValues = new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
                    assertEquals(isRelatedTo.get(isRelatedToEntry.getKey()).size(), isRelatedToValues.size());
                    for (String v : isRelatedToEntry.getValue()) {
                        assertTrue(isRelatedToValues.contains(v));
                    }
                }
                // RelatesTo
                for (Map.Entry<String, Set<String>> relatesToEntry : relatesTo.entrySet()) {
                    String compoundValue = EntityColumnPrefix.RELATES_TO.readResult(result, relatesToEntry.getKey()).toString();
                    // id3?id4?id5
                    Set<String> relatesToValues = new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
                    assertEquals(relatesTo.get(relatesToEntry.getKey()).size(), relatesToValues.size());
                    for (String v : relatesToEntry.getValue()) {
                        assertTrue(relatesToValues.contains(v));
                    }
                }
                // Configuration
                Map<String, Object> configColumns = EntityColumnPrefix.CONFIG.readResults(result, stringKeyConverter);
                assertEquals(conf, configColumns);
                NavigableMap<String, NavigableMap<Long, Number>> metricsResult = EntityColumnPrefix.METRIC.readResultsWithTimestamps(result, stringKeyConverter);
                NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
                matchMetrics(metricValues, metricMap);
            }
        }
        assertEquals(1, rowCount);
        assertEquals(16, colCount);
        // read the timeline entity using the reader this time
        TimelineEntity e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), Integer.MAX_VALUE));
        Set<TimelineEntity> es1 = reader.getEntities(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), Integer.MAX_VALUE));
        assertNotNull(e1);
        assertEquals(1, es1.size());
        // verify attributes
        assertEquals(id, e1.getId());
        assertEquals(type, e1.getType());
        assertEquals(cTime, e1.getCreatedTime());
        Map<String, Object> infoMap2 = e1.getInfo();
        assertEquals(infoMap, infoMap2);
        Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
        assertEquals(isRelatedTo, isRelatedTo2);
        Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
        assertEquals(relatesTo, relatesTo2);
        Map<String, String> conf2 = e1.getConfigs();
        assertEquals(conf, conf2);
        Set<TimelineMetric> metrics2 = e1.getMetrics();
        assertEquals(metrics, metrics2);
        for (TimelineMetric metric2 : metrics2) {
            Map<Long, Number> metricValues2 = metric2.getValues();
            matchMetrics(metricValues, metricValues2);
        }
        e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        assertNotNull(e1);
        assertEquals(id, e1.getId());
        assertEquals(type, e1.getType());
        assertEquals(cTime, e1.getCreatedTime());
        assertEquals(infoMap, e1.getInfo());
        assertEquals(isRelatedTo, e1.getIsRelatedToEntities());
        assertEquals(relatesTo, e1.getRelatesToEntities());
        assertEquals(conf, e1.getConfigs());
        for (TimelineMetric metric : e1.getMetrics()) {
            assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric.getType());
            assertEquals(1, metric.getValues().size());
            assertTrue(metric.getValues().containsKey(ts - 20000));
            assertEquals(metricValues.get(ts - 20000), metric.getValues().get(ts - 20000));
        }
    } finally {
        if (hbi != null) {
            hbi.stop();
            hbi.close();
        }
    }
}

Example 49 with Result

use of org.apache.hadoop.hbase.client.Result in project hbase by apache.

the class ResponseConverter method getResults.

/**
   * Create Results from the cells using the cells meta data.
   * @param cellScanner
   * @param response
   * @return results
   */
public static Result[] getResults(CellScanner cellScanner, ScanResponse response) throws IOException {
    if (response == null)
        return null;
    // If cellscanner, then the number of Results to return is the count of elements in the
    // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
    int noOfResults = cellScanner != null ? response.getCellsPerResultCount() : response.getResultsCount();
    Result[] results = new Result[noOfResults];
    for (int i = 0; i < noOfResults; i++) {
        if (cellScanner != null) {
            // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
            // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
            int noOfCells = response.getCellsPerResult(i);
            boolean isPartial = response.getPartialFlagPerResultCount() > i ? response.getPartialFlagPerResult(i) : false;
            List<Cell> cells = new ArrayList<>(noOfCells);
            for (int j = 0; j < noOfCells; j++) {
                try {
                    if (cellScanner.advance() == false) {
                        // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
                        // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
                        // same scanner will result in OutOfOrderScannerNextException
                        String msg = "Results sent from server=" + noOfResults + ". But only got " + i + " results completely at client. Resetting the scanner to scan again.";
                        LOG.error(msg);
                        throw new DoNotRetryIOException(msg);
                    }
                } catch (IOException ioe) {
                    // We are getting IOE while retrieving the cells for Results.
                    // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
                    // same scanner will result in OutOfOrderScannerNextException
                    LOG.error("Exception while reading cells from result." + "Resetting the scanner to scan again.", ioe);
                    throw new DoNotRetryIOException("Resetting the scanner.", ioe);
                }
                cells.add(cellScanner.current());
            }
            results[i] = Result.create(cells, null, response.getStale(), isPartial);
        } else {
            // Result is pure pb.
            results[i] = ProtobufUtil.toResult(response.getResults(i));
        }
    }
    return results;
}

Example 50 with Result

use of org.apache.hadoop.hbase.client.Result in project hbase by apache.

the class ScannerResultGenerator method next.

public Cell next() {
    if (cache != null) {
        Cell kv = cache;
        cache = null;
        return kv;
    }
    boolean loop;
    do {
        loop = false;
        if (rowI != null) {
            if (rowI.hasNext()) {
                return rowI.next();
            } else {
                rowI = null;
            }
        }
        if (cached != null) {
            rowI = cached.listCells().iterator();
            loop = true;
            cached = null;
        } else {
            Result result = null;
            try {
                result = scanner.next();
            } catch (UnknownScannerException e) {
                throw new IllegalArgumentException(e);
            } catch (TableNotEnabledException tnee) {
                throw new IllegalStateException(tnee);
            } catch (TableNotFoundException tnfe) {
                throw new IllegalArgumentException(tnfe);
            } catch (IOException e) {
                LOG.error(StringUtils.stringifyException(e));
            }
            if (result != null && !result.isEmpty()) {
                rowI = result.listCells().iterator();
                loop = true;
            }
        }
    } while (loop);
    return null;
}