Examples with SegmentMetadata com.linkedin.pinot.common.segment.SegmentMetadata used on opensource projects

Example 26 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class HelixBrokerStarterTest method addOneSegment.

private void addOneSegment(String tableName) {
    final SegmentMetadata segmentMetadata = new SimpleSegmentMetadata(tableName);
    LOGGER.info("Trying to add IndexSegment : " + segmentMetadata.getName());
    _pinotResourceManager.addSegment(segmentMetadata, "http://localhost:something");
}

Example 27 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithDaysTimeUnit.

/**
   * Test with daysSinceEpoch time unit and make 10 segments with expired time value, 10 segments
   * with the day after tomorrow's time stamp.
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithDaysTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("15544", "15544", TimeUnit.DAYS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(theDayAfterTomorrowSinceEpoch + "", theDayAfterTomorrowSinceEpoch + "", TimeUnit.DAYS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}

Example 28 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class SegmentAssignmentStrategyTest method addOneSegment.

private void addOneSegment(String tableName) {
    final SegmentMetadata segmentMetadata = new SimpleSegmentMetadata(tableName);
    LOGGER.info("Trying to add IndexSegment : " + segmentMetadata.getName());
    _pinotHelixResourceManager.addSegment(segmentMetadata, "downloadUrl");
}

Example 29 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class PinotResourceManagerTest method testDeletingTheSameSegmentInSegmentDeletionManager.

@Test
public void testDeletingTheSameSegmentInSegmentDeletionManager() throws Exception {
    final SegmentMetadata segmentMetadata = new SimpleSegmentMetadata(TABLE_NAME);
    final String segmentName = segmentMetadata.getName();
    File segmentFile = new File(LOCAL_DISK_DIR + "/" + TABLE_NAME + "/" + segmentName);
    for (int i = 0; i < 2; i++) {
        addOneSegment(TABLE_NAME);
        // Waiting for the external view to update
        Thread.sleep(2000);
        final ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TABLE_NAME));
        List<String> segmentsList = new ArrayList<>(externalView.getPartitionSet().size());
        segmentsList.addAll(externalView.getPartitionSet());
        _pinotHelixResourceManager.deleteSegments(TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TABLE_NAME), segmentsList);
    }
    assert (!segmentFile.exists());
}

Example 30 with SegmentMetadata

use of com.linkedin.pinot.common.segment.SegmentMetadata in project pinot by linkedin.

the class ValidationManagerTest method testTotalDocumentCountRealTime.

@Test
public void testTotalDocumentCountRealTime() throws Exception {
    // Create a bunch of dummy segments
    String testTableName = "TestTableTotalDocCountTest";
    final String group1 = testTableName + "_REALTIME_1466446700000_34";
    final String group2 = testTableName + "_REALTIME_1466446700000_17";
    String segmentName1 = new HLCSegmentName(group1, "0", "1").getSegmentName();
    String segmentName2 = new HLCSegmentName(group1, "0", "2").getSegmentName();
    String segmentName3 = new HLCSegmentName(group1, "0", "3").getSegmentName();
    String segmentName4 = new HLCSegmentName(group2, "0", "3").getSegmentName();
    DummyMetadata metadata1 = new DummyMetadata(testTableName, segmentName1, 10);
    DummyMetadata metadata2 = new DummyMetadata(testTableName, segmentName2, 20);
    DummyMetadata metadata3 = new DummyMetadata(testTableName, segmentName3, 30);
    // This should get ignored in the count as it belongs to a different group id
    DummyMetadata metadata4 = new DummyMetadata(testTableName, segmentName4, 20);
    // Add them to a list
    List<SegmentMetadata> segmentMetadataList = new ArrayList<SegmentMetadata>();
    segmentMetadataList.add(metadata1);
    segmentMetadataList.add(metadata2);
    segmentMetadataList.add(metadata3);
    segmentMetadataList.add(metadata4);
    Assert.assertEquals(ValidationManager.computeRealtimeTotalDocumentInSegments(segmentMetadataList, true), 60);
    // Now add some low level segment names
    String segmentName5 = new LLCSegmentName(testTableName, 1, 0, 1000).getSegmentName();
    String segmentName6 = new LLCSegmentName(testTableName, 2, 27, 10000).getSegmentName();
    DummyMetadata metadata5 = new DummyMetadata(testTableName, segmentName5, 10);
    DummyMetadata metadata6 = new DummyMetadata(testTableName, segmentName6, 5);
    segmentMetadataList.add(metadata5);
    segmentMetadataList.add(metadata6);
    // Only the LLC segments should get counted.
    Assert.assertEquals(ValidationManager.computeRealtimeTotalDocumentInSegments(segmentMetadataList, false), 15);
}