Examples with HistoryRecord io.pravega.controller.store.stream.tables.HistoryRecord used on opensource projects

Example 6 with HistoryRecord

use of io.pravega.controller.store.stream.tables.HistoryRecord in project pravega by pravega.

the class TableHelperTest method predecessorAndSuccessorTest.

@Test
public void predecessorAndSuccessorTest() {
    // multiple rows in history table, find predecessor
    // - more than one predecessor
    // - one predecessor
    // - no predecessor
    // - immediate predecessor
    // - predecessor few rows behind
    List<Segment> segments = new ArrayList<>();
    List<Integer> newSegments = Lists.newArrayList(0, 1, 2, 3, 4);
    long timestamp = System.currentTimeMillis();
    Segment zero = new Segment(0, timestamp, 0, 0.2);
    segments.add(zero);
    Segment one = new Segment(1, timestamp, 0.2, 0.4);
    segments.add(one);
    Segment two = new Segment(2, timestamp, 0.4, 0.6);
    segments.add(two);
    Segment three = new Segment(3, timestamp, 0.6, 0.8);
    segments.add(three);
    Segment four = new Segment(4, timestamp, 0.8, 1);
    segments.add(four);
    List<Integer> predecessors, successors;
    // find predecessors and successors when update to history and index table hasnt happened
    predecessors = TableHelper.getOverlaps(zero, TableHelper.findSegmentPredecessorCandidates(zero, new byte[0], new byte[0]).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(zero, TableHelper.findSegmentSuccessorCandidates(zero, new byte[0], new byte[0]).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, new ArrayList<Integer>());
    assertEquals(successors, new ArrayList<Integer>());
    byte[] historyTable = TableHelper.createHistoryTable(timestamp, newSegments);
    byte[] indexTable = TableHelper.createIndexTable(timestamp, 0);
    int nextHistoryOffset = historyTable.length;
    // 3, 4 -> 5
    newSegments = Lists.newArrayList(0, 1, 2, 5);
    timestamp = timestamp + 1;
    Segment five = new Segment(5, timestamp, 0.6, 1);
    segments.add(five);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments);
    // check predecessor segment in partial record
    predecessors = TableHelper.getOverlaps(five, TableHelper.findSegmentPredecessorCandidates(five, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    // check that segment from partial record is returned as successor
    successors = TableHelper.getOverlaps(three, TableHelper.findSegmentSuccessorCandidates(three, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(3, 4));
    assertEquals(successors, Lists.newArrayList(5));
    HistoryRecord partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    // Notice: segment was created at timestamp but we are recording its entry in history table at timestamp + 5
    timestamp = timestamp + 5;
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp);
    indexTable = TableHelper.updateIndexTable(indexTable, timestamp, nextHistoryOffset);
    nextHistoryOffset = historyTable.length;
    // 1 -> 6,7.. 2,5 -> 8
    newSegments = Lists.newArrayList(0, 6, 7, 8);
    timestamp = timestamp + 10;
    Segment six = new Segment(6, timestamp, 0.2, 0.3);
    segments.add(six);
    Segment seven = new Segment(7, timestamp, 0.3, 0.4);
    segments.add(seven);
    Segment eight = new Segment(8, timestamp, 0.4, 1);
    segments.add(eight);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments);
    // check that previous partial record is not a regular record and its successor and predecessors are returned successfully
    predecessors = TableHelper.getOverlaps(five, TableHelper.findSegmentPredecessorCandidates(five, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(five, TableHelper.findSegmentSuccessorCandidates(five, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(3, 4));
    assertEquals(successors, Lists.newArrayList(8));
    partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    timestamp = timestamp + 5;
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp);
    indexTable = TableHelper.updateIndexTable(indexTable, timestamp, nextHistoryOffset);
    nextHistoryOffset = historyTable.length;
    // 7 -> 9,10.. 8 -> 10, 11
    newSegments = Lists.newArrayList(0, 6, 9, 10, 11);
    timestamp = timestamp + 10;
    Segment nine = new Segment(9, timestamp, 0.3, 0.35);
    segments.add(nine);
    Segment ten = new Segment(10, timestamp, 0.35, 0.6);
    segments.add(ten);
    Segment eleven = new Segment(11, timestamp, 0.6, 1);
    segments.add(eleven);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments);
    partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    timestamp = timestamp + 5;
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp);
    // find predecessor and successor with index table being stale
    predecessors = TableHelper.getOverlaps(ten, TableHelper.findSegmentPredecessorCandidates(ten, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(seven, TableHelper.findSegmentSuccessorCandidates(seven, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(7, 8));
    assertEquals(successors, Lists.newArrayList(9, 10));
    indexTable = TableHelper.updateIndexTable(indexTable, timestamp, nextHistoryOffset);
    // 0 has no successor and no predecessor
    // 10 has multiple predecessor
    // 1 has a successor few rows down
    predecessors = TableHelper.getOverlaps(zero, TableHelper.findSegmentPredecessorCandidates(zero, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(zero, TableHelper.findSegmentSuccessorCandidates(zero, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, new ArrayList<Integer>());
    assertEquals(successors, new ArrayList<Integer>());
    predecessors = TableHelper.getOverlaps(one, TableHelper.findSegmentPredecessorCandidates(one, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(one, TableHelper.findSegmentSuccessorCandidates(one, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, new ArrayList<Integer>());
    assertEquals(successors, Lists.newArrayList(6, 7));
    predecessors = TableHelper.getOverlaps(two, TableHelper.findSegmentPredecessorCandidates(two, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(two, TableHelper.findSegmentSuccessorCandidates(two, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, new ArrayList<Integer>());
    assertEquals(successors, Lists.newArrayList(8));
    predecessors = TableHelper.getOverlaps(three, TableHelper.findSegmentPredecessorCandidates(three, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(three, TableHelper.findSegmentSuccessorCandidates(three, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, new ArrayList<Integer>());
    assertEquals(successors, Lists.newArrayList(5));
    predecessors = TableHelper.getOverlaps(four, TableHelper.findSegmentPredecessorCandidates(four, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(four, TableHelper.findSegmentSuccessorCandidates(four, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, new ArrayList<Integer>());
    assertEquals(successors, Lists.newArrayList(5));
    predecessors = TableHelper.getOverlaps(five, TableHelper.findSegmentPredecessorCandidates(five, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(five, TableHelper.findSegmentSuccessorCandidates(five, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(3, 4));
    assertEquals(successors, Lists.newArrayList(8));
    predecessors = TableHelper.getOverlaps(six, TableHelper.findSegmentPredecessorCandidates(six, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(six, TableHelper.findSegmentSuccessorCandidates(six, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(1));
    assertEquals(successors, new ArrayList<>());
    predecessors = TableHelper.getOverlaps(seven, TableHelper.findSegmentPredecessorCandidates(seven, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(seven, TableHelper.findSegmentSuccessorCandidates(seven, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(1));
    assertEquals(successors, Lists.newArrayList(9, 10));
    predecessors = TableHelper.getOverlaps(eight, TableHelper.findSegmentPredecessorCandidates(eight, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(eight, TableHelper.findSegmentSuccessorCandidates(eight, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(2, 5));
    assertEquals(successors, Lists.newArrayList(10, 11));
    predecessors = TableHelper.getOverlaps(nine, TableHelper.findSegmentPredecessorCandidates(nine, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(nine, TableHelper.findSegmentSuccessorCandidates(nine, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(7));
    assertEquals(successors, new ArrayList<>());
    predecessors = TableHelper.getOverlaps(ten, TableHelper.findSegmentPredecessorCandidates(ten, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(ten, TableHelper.findSegmentSuccessorCandidates(ten, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(7, 8));
    assertEquals(successors, new ArrayList<>());
    predecessors = TableHelper.getOverlaps(eleven, TableHelper.findSegmentPredecessorCandidates(eleven, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(eleven, TableHelper.findSegmentSuccessorCandidates(eleven, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(8));
    assertEquals(successors, new ArrayList<>());
}

Example 7 with HistoryRecord

use of io.pravega.controller.store.stream.tables.HistoryRecord in project pravega by pravega.

the class TableHelperTest method truncationTest.

@Test(timeout = 10000)
public void truncationTest() {
    final List<Integer> startSegments = Lists.newArrayList(0, 1);
    // epoch 0
    long timestamp = System.currentTimeMillis();
    byte[] segmentTable = createSegmentTable(2, timestamp);
    byte[] historyTable = TableHelper.createHistoryTable(timestamp, startSegments);
    byte[] indexTable = TableHelper.createIndexTable(timestamp, 0);
    List<Integer> activeSegments = TableHelper.getActiveSegments(historyTable);
    assertEquals(activeSegments, startSegments);
    // epoch 1
    List<Integer> newSegments1 = Lists.newArrayList(0, 2, 3);
    List<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>(0.5, 0.75));
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>(0.75, 1.0));
    segmentTable = updateSegmentTable(segmentTable, newRanges, timestamp + 1);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments1);
    HistoryRecord partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp + 1);
    indexTable = TableHelper.updateIndexTable(indexTable, timestamp + 1, partial.getOffset());
    // epoch 2
    List<Integer> newSegments2 = Lists.newArrayList(0, 2, 4, 5);
    newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>(0.75, (0.75 + 1.0) / 2));
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>((0.75 + 1.0) / 2, 1.0));
    segmentTable = updateSegmentTable(segmentTable, newRanges, timestamp + 2);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments2);
    partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp + 2);
    indexTable = TableHelper.updateIndexTable(indexTable, timestamp + 2, partial.getOffset());
    // epoch 3
    List<Integer> newSegments3 = Lists.newArrayList(0, 4, 5, 6, 7);
    newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>(0.5, (0.75 + 0.5) / 2));
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>((0.75 + 0.5) / 2, 0.75));
    segmentTable = updateSegmentTable(segmentTable, newRanges, timestamp + 3);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments3);
    partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp + 3);
    indexTable = TableHelper.updateIndexTable(indexTable, timestamp + 3, partial.getOffset());
    // epoch 4
    List<Integer> newSegments4 = Lists.newArrayList(4, 5, 6, 7, 8, 9);
    newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>(0.0, (0.0 + 0.5) / 2));
    newRanges.add(new AbstractMap.SimpleEntry<Double, Double>((0.0 + 0.5) / 2, 0.5));
    segmentTable = updateSegmentTable(segmentTable, newRanges, timestamp + 4);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments4);
    partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp + 4);
    indexTable = TableHelper.updateIndexTable(indexTable, timestamp + 4, partial.getOffset());
    // happy day
    Map<Integer, Long> streamCut1 = new HashMap<>();
    streamCut1.put(0, 1L);
    streamCut1.put(1, 1L);
    StreamTruncationRecord truncationRecord = TableHelper.computeTruncationRecord(indexTable, historyTable, segmentTable, streamCut1, StreamTruncationRecord.EMPTY);
    assertTrue(truncationRecord.getToDelete().isEmpty());
    assertTrue(truncationRecord.getStreamCut().equals(streamCut1));
    assertTrue(truncationRecord.getCutEpochMap().get(0) == 0 && truncationRecord.getCutEpochMap().get(1) == 0);
    truncationRecord = truncationRecord.mergeDeleted();
    Map<Integer, Long> streamCut2 = new HashMap<>();
    streamCut2.put(0, 1L);
    streamCut2.put(2, 1L);
    streamCut2.put(4, 1L);
    streamCut2.put(5, 1L);
    truncationRecord = TableHelper.computeTruncationRecord(indexTable, historyTable, segmentTable, streamCut2, truncationRecord);
    assertTrue(truncationRecord.getToDelete().size() == 2 && truncationRecord.getToDelete().contains(1) && truncationRecord.getToDelete().contains(3));
    assertTrue(truncationRecord.getStreamCut().equals(streamCut2));
    assertTrue(truncationRecord.getCutEpochMap().get(0) == 2 && truncationRecord.getCutEpochMap().get(2) == 2 && truncationRecord.getCutEpochMap().get(4) == 2 && truncationRecord.getCutEpochMap().get(5) == 2);
    truncationRecord = truncationRecord.mergeDeleted();
    Map<Integer, Long> streamCut3 = new HashMap<>();
    streamCut3.put(2, 10L);
    streamCut3.put(4, 10L);
    streamCut3.put(5, 10L);
    streamCut3.put(8, 10L);
    streamCut3.put(9, 10L);
    truncationRecord = TableHelper.computeTruncationRecord(indexTable, historyTable, segmentTable, streamCut3, truncationRecord);
    assertTrue(truncationRecord.getToDelete().size() == 1 && truncationRecord.getToDelete().contains(0));
    assertTrue(truncationRecord.getStreamCut().equals(streamCut3));
    assertTrue(truncationRecord.getCutEpochMap().get(2) == 2 && truncationRecord.getCutEpochMap().get(4) == 4 && truncationRecord.getCutEpochMap().get(5) == 4 && truncationRecord.getCutEpochMap().get(8) == 4 && truncationRecord.getCutEpochMap().get(9) == 4);
    truncationRecord = truncationRecord.mergeDeleted();
    // behind previous
    Map<Integer, Long> streamCut4 = new HashMap<>();
    streamCut4.put(2, 1L);
    streamCut4.put(4, 1L);
    streamCut4.put(5, 1L);
    streamCut4.put(8, 1L);
    streamCut4.put(9, 1L);
    byte[] finalIndexTable = indexTable;
    byte[] finalHistoryTable = historyTable;
    byte[] finalSegmentTable = segmentTable;
    StreamTruncationRecord finalTruncationRecord = truncationRecord;
    AssertExtensions.assertThrows("", () -> TableHelper.computeTruncationRecord(finalIndexTable, finalHistoryTable, finalSegmentTable, streamCut4, finalTruncationRecord), e -> e instanceof IllegalArgumentException);
    Map<Integer, Long> streamCut5 = new HashMap<>();
    streamCut3.put(2, 10L);
    streamCut3.put(4, 10L);
    streamCut3.put(5, 10L);
    streamCut3.put(0, 10L);
    AssertExtensions.assertThrows("", () -> TableHelper.computeTruncationRecord(finalIndexTable, finalHistoryTable, finalSegmentTable, streamCut5, finalTruncationRecord), e -> e instanceof IllegalArgumentException);
}

Example 8 with HistoryRecord

use of io.pravega.controller.store.stream.tables.HistoryRecord in project pravega by pravega.

the class TableHelperTest method testNoValuePresentError.

@Test
public void testNoValuePresentError() {
    // no value present error comes because:
    // - Index is not yet updated.
    // - And segment creation time is before history record's time.
    // While trying to find successor we look for record in history table with
    // segment creation and get an old record. We search for segment sealed event
    // between history record and last indexed entry both of which preceed segment creation entry.
    List<Segment> segments = new ArrayList<>();
    List<Integer> newSegments = Lists.newArrayList(0, 1, 2, 3, 4);
    long timestamp = System.currentTimeMillis();
    Segment zero = new Segment(0, timestamp, 0, 0.2);
    segments.add(zero);
    Segment one = new Segment(1, timestamp, 0.2, 0.4);
    segments.add(one);
    Segment two = new Segment(2, timestamp, 0.4, 0.6);
    segments.add(two);
    Segment three = new Segment(3, timestamp, 0.6, 0.8);
    segments.add(three);
    Segment four = new Segment(4, timestamp, 0.8, 1);
    segments.add(four);
    byte[] historyTable = TableHelper.createHistoryTable(timestamp, newSegments);
    byte[] indexTable = TableHelper.createIndexTable(timestamp, 0);
    timestamp = timestamp + 10000;
    // scale down
    Segment five = new Segment(5, timestamp, 0.4, 1);
    segments.add(five);
    newSegments = Lists.newArrayList(0, 1, 5);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments);
    HistoryRecord partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    // Notice: segment was created at timestamp but we are recording its entry in history table at timestamp + 10000
    timestamp = timestamp + 10000;
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp);
    timestamp = timestamp + 10000;
    Segment six = new Segment(6, timestamp, 0.0, 1);
    segments.add(six);
    newSegments = Lists.newArrayList(6);
    historyTable = TableHelper.addPartialRecordToHistoryTable(historyTable, newSegments);
    partial = HistoryRecord.readLatestRecord(historyTable, false).get();
    // Notice: segment was created at timestamp but we are recording its entry in history table at timestamp + 10000
    timestamp = timestamp + 10000;
    historyTable = TableHelper.completePartialRecordInHistoryTable(historyTable, partial, timestamp);
    List<Integer> predecessors, successors;
    // find predecessors and successors when update to index table hasn't happened
    predecessors = TableHelper.getOverlaps(five, TableHelper.findSegmentPredecessorCandidates(five, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    successors = TableHelper.getOverlaps(five, TableHelper.findSegmentSuccessorCandidates(five, indexTable, historyTable).stream().map(x -> getSegment(x, segments)).collect(Collectors.toList()));
    assertEquals(predecessors, Lists.newArrayList(2, 3, 4));
    assertEquals(successors, Lists.newArrayList(6));
}