Examples with LongConverter org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter used on opensource projects

Example 1 with LongConverter

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter in project hadoop by apache.

the class TestHBaseStorageFlowRunCompaction method testProcessSummationMoreCellsSumFinal.

@Test
public void testProcessSummationMoreCellsSumFinal() throws IOException {
    FlowScanner fs = getFlowScannerForTestingCompaction();
    // note down the current timestamp
    long currentTimestamp = System.currentTimeMillis();
    long cellValue1 = 1236L;
    long cellValue2 = 28L;
    List<Tag> tags = new ArrayList<>();
    Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(), "application_1234588888_999888");
    tags.add(t);
    byte[] tagByteArray = Tag.fromList(tags);
    SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
    // create a cell with a VERY old timestamp and attribute SUM_FINAL
    Cell c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, 120L, Bytes.toBytes(cellValue1), tagByteArray);
    currentColumnCells.add(c1);
    tags = new ArrayList<>();
    t = new Tag(AggregationOperation.SUM.getTagType(), "application_100000000001_119101");
    tags.add(t);
    tagByteArray = Tag.fromList(tags);
    // create a cell with a VERY old timestamp but has attribute SUM
    Cell c2 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, 130L, Bytes.toBytes(cellValue2), tagByteArray);
    currentColumnCells.add(c2);
    List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells, new LongConverter(), currentTimestamp);
    assertNotNull(cells);
    // we should be getting back two cells
    // one is the flow sum cell
    // another is the cell with SUM attribute
    assertEquals(2, cells.size());
    Cell returnedCell = cells.get(0);
    assertNotNull(returnedCell);
    long inputTs1 = c1.getTimestamp();
    long inputTs2 = c2.getTimestamp();
    long returnTs = returnedCell.getTimestamp();
    long returnValue = Bytes.toLong(CellUtil.cloneValue(returnedCell));
    // current timestamp
    if (returnValue == cellValue2) {
        assertTrue(returnTs == inputTs2);
    } else if (returnValue == cellValue1) {
        assertTrue(returnTs >= currentTimestamp);
        assertTrue(returnTs != inputTs1);
    } else {
        // raise a failure since we expect only these two values back
        Assert.fail();
    }
}

Example 2 with LongConverter

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter in project hadoop by apache.

the class TestHBaseStorageFlowRunCompaction method testProcessSummationOneCellSumFinal.

@Test
public void testProcessSummationOneCellSumFinal() throws IOException {
    FlowScanner fs = getFlowScannerForTestingCompaction();
    // note down the current timestamp
    long currentTimestamp = System.currentTimeMillis();
    List<Tag> tags = new ArrayList<>();
    Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(), "application_123458888888_999888");
    tags.add(t);
    byte[] tagByteArray = Tag.fromList(tags);
    SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
    // create a cell with a VERY old timestamp
    Cell c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, 120L, Bytes.toBytes(1110L), tagByteArray);
    currentColumnCells.add(c1);
    List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells, new LongConverter(), currentTimestamp);
    assertNotNull(cells);
    // we should not get the same cell back
    // but we get back the flow cell
    assertEquals(1, cells.size());
    Cell returnedCell = cells.get(0);
    // it's NOT the same cell
    assertNotEquals(c1, returnedCell);
    long inputTs = c1.getTimestamp();
    long returnTs = returnedCell.getTimestamp();
    // the returned Ts will be far greater than input ts as well as the noted
    // current timestamp
    assertTrue(returnTs > inputTs);
    assertTrue(returnTs >= currentTimestamp);
}

Example 3 with LongConverter

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter in project hadoop by apache.

the class TestHBaseStorageFlowRunCompaction method checkProcessSummationMoreCellsSumFinal2.

@Test
public void checkProcessSummationMoreCellsSumFinal2() throws IOException {
    long cellValue1 = 1236L;
    long cellValue2 = 28L;
    long cellValue3 = 1236L;
    long cellValue4 = 1236L;
    FlowScanner fs = getFlowScannerForTestingCompaction();
    // note down the current timestamp
    long currentTimestamp = System.currentTimeMillis();
    long cell1Ts = 1200120L;
    long cell2Ts = TimestampGenerator.getSupplementedTimestamp(System.currentTimeMillis(), "application_123746661110_11202");
    long cell3Ts = 1277719L;
    long cell4Ts = currentTimestamp - 10;
    SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
    List<Tag> tags = new ArrayList<>();
    Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(), "application_1234588888_91188");
    tags.add(t);
    byte[] tagByteArray = Tag.fromList(tags);
    // create a cell with a VERY old timestamp and attribute SUM_FINAL
    Cell c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, cell1Ts, Bytes.toBytes(cellValue1), tagByteArray);
    currentColumnCells.add(c1);
    tags = new ArrayList<>();
    t = new Tag(AggregationOperation.SUM_FINAL.getTagType(), "application_12700000001_29102");
    tags.add(t);
    tagByteArray = Tag.fromList(tags);
    // create a cell with a recent timestamp and attribute SUM_FINAL
    Cell c2 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, cell2Ts, Bytes.toBytes(cellValue2), tagByteArray);
    currentColumnCells.add(c2);
    tags = new ArrayList<>();
    t = new Tag(AggregationOperation.SUM.getTagType(), "application_191780000000001_8195");
    tags.add(t);
    tagByteArray = Tag.fromList(tags);
    // create a cell with a VERY old timestamp but has attribute SUM
    Cell c3 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, cell3Ts, Bytes.toBytes(cellValue3), tagByteArray);
    currentColumnCells.add(c3);
    tags = new ArrayList<>();
    t = new Tag(AggregationOperation.SUM.getTagType(), "application_191780000000001_98104");
    tags.add(t);
    tagByteArray = Tag.fromList(tags);
    // create a cell with a VERY old timestamp but has attribute SUM
    Cell c4 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, cell4Ts, Bytes.toBytes(cellValue4), tagByteArray);
    currentColumnCells.add(c4);
    List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells, new LongConverter(), currentTimestamp);
    assertNotNull(cells);
    // we should be getting back 4 cells
    // one is the flow sum cell
    // two are the cells with SUM attribute
    // one cell with SUM_FINAL
    assertEquals(4, cells.size());
    for (int i = 0; i < cells.size(); i++) {
        Cell returnedCell = cells.get(0);
        assertNotNull(returnedCell);
        long returnTs = returnedCell.getTimestamp();
        long returnValue = Bytes.toLong(CellUtil.cloneValue(returnedCell));
        if (returnValue == cellValue2) {
            assertTrue(returnTs == cell2Ts);
        } else if (returnValue == cellValue3) {
            assertTrue(returnTs == cell3Ts);
        } else if (returnValue == cellValue4) {
            assertTrue(returnTs == cell4Ts);
        } else if (returnValue == cellValue1) {
            assertTrue(returnTs != cell1Ts);
            assertTrue(returnTs > cell1Ts);
            assertTrue(returnTs >= currentTimestamp);
        } else {
            // raise a failure since we expect only these two values back
            Assert.fail();
        }
    }
}

Example 4 with LongConverter

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter in project hadoop by apache.

the class TestHBaseStorageFlowRunCompaction method testProcessSummationEmpty.

@Test
public void testProcessSummationEmpty() throws IOException {
    FlowScanner fs = getFlowScannerForTestingCompaction();
    long currentTimestamp = System.currentTimeMillis();
    LongConverter longConverter = new LongConverter();
    SortedSet<Cell> currentColumnCells = null;
    List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells, longConverter, currentTimestamp);
    assertNotNull(cells);
    assertEquals(0, cells.size());
    currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
    cells = fs.processSummationMajorCompaction(currentColumnCells, longConverter, currentTimestamp);
    assertNotNull(cells);
    assertEquals(0, cells.size());
}

Example 5 with LongConverter

use of org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter in project hadoop by apache.

the class TestHBaseStorageFlowRunCompaction method checkProcessSummationMoreCellsSumFinalVariedTags.

// tests with many cells
// of type SUM and SUM_FINAL
// NOT cells of SUM_FINAL will expire
@Test
public void checkProcessSummationMoreCellsSumFinalVariedTags() throws IOException {
    FlowScanner fs = getFlowScannerForTestingCompaction();
    int countFinal = 20100;
    int countNotFinal = 1000;
    int countFinalNotExpire = 7009;
    long cellValueFinal = 1000L;
    long cellValueNotFinal = 28L;
    // note down the current timestamp
    long currentTimestamp = System.currentTimeMillis();
    long cellTsFinalStart = 10001120L;
    long cellTsFinal = cellTsFinalStart;
    long cellTsFinalStartNotExpire = TimestampGenerator.getSupplementedTimestamp(System.currentTimeMillis(), "application_10266666661166_118821");
    long cellTsFinalNotExpire = cellTsFinalStartNotExpire;
    long cellTsNotFinalStart = currentTimestamp - 5;
    long cellTsNotFinal = cellTsNotFinalStart;
    SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
    List<Tag> tags = null;
    Tag t = null;
    Cell c1 = null;
    // insert SUM_FINAL cells which will expire
    for (int i = 0; i < countFinal; i++) {
        tags = new ArrayList<>();
        t = new Tag(AggregationOperation.SUM_FINAL.getTagType(), "application_123450000" + i + "01_19" + i);
        tags.add(t);
        byte[] tagByteArray = Tag.fromList(tags);
        // create a cell with a VERY old timestamp and attribute SUM_FINAL
        c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, cellTsFinal, Bytes.toBytes(cellValueFinal), tagByteArray);
        currentColumnCells.add(c1);
        cellTsFinal++;
    }
    // insert SUM_FINAL cells which will NOT expire
    for (int i = 0; i < countFinalNotExpire; i++) {
        tags = new ArrayList<>();
        t = new Tag(AggregationOperation.SUM_FINAL.getTagType(), "application_123450000" + i + "01_19" + i);
        tags.add(t);
        byte[] tagByteArray = Tag.fromList(tags);
        // create a cell with a VERY old timestamp and attribute SUM_FINAL
        c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, cellTsFinalNotExpire, Bytes.toBytes(cellValueFinal), tagByteArray);
        currentColumnCells.add(c1);
        cellTsFinalNotExpire++;
    }
    // add SUM cells
    for (int i = 0; i < countNotFinal; i++) {
        tags = new ArrayList<>();
        t = new Tag(AggregationOperation.SUM.getTagType(), "application_1987650000" + i + "83_911" + i);
        tags.add(t);
        byte[] tagByteArray = Tag.fromList(tags);
        // create a cell with attribute SUM
        c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier, cellTsNotFinal, Bytes.toBytes(cellValueNotFinal), tagByteArray);
        currentColumnCells.add(c1);
        cellTsNotFinal++;
    }
    List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells, new LongConverter(), currentTimestamp);
    assertNotNull(cells);
    // we should be getting back
    // countNotFinal + countFinalNotExpire + 1 cells
    // one is the flow sum cell
    // count = the cells with SUM attribute
    // count = the cells with SUM_FINAL attribute but not expired
    assertEquals(countFinalNotExpire + countNotFinal + 1, cells.size());
    for (int i = 0; i < cells.size(); i++) {
        Cell returnedCell = cells.get(0);
        assertNotNull(returnedCell);
        long returnTs = returnedCell.getTimestamp();
        long returnValue = Bytes.toLong(CellUtil.cloneValue(returnedCell));
        if (returnValue == (countFinal * cellValueFinal)) {
            assertTrue(returnTs > (cellTsFinalStart + countFinal));
            assertTrue(returnTs >= currentTimestamp);
        } else if (returnValue == cellValueNotFinal) {
            assertTrue(returnTs >= cellTsNotFinalStart);
            assertTrue(returnTs <= cellTsNotFinalStart + countNotFinal);
        } else if (returnValue == cellValueFinal) {
            assertTrue(returnTs >= cellTsFinalStartNotExpire);
            assertTrue(returnTs <= cellTsFinalStartNotExpire + countFinalNotExpire);
        } else {
            // raise a failure since we expect only these values back
            Assert.fail();
        }
    }
}