Examples with OfflineSegmentZKMetadata com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata used on opensource projects

Example 16 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class RoutingTableTest method testTimeBoundaryRegression.

@Test
public void testTimeBoundaryRegression() throws Exception {
    final FakePropertyStore propertyStore = new FakePropertyStore();
    final OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
    offlineSegmentZKMetadata.setTimeUnit(TimeUnit.DAYS);
    offlineSegmentZKMetadata.setEndTime(1234L);
    propertyStore.setContents(ZKMetadataProvider.constructPropertyStorePathForSegment("myTable_OFFLINE", "someSegment_0"), offlineSegmentZKMetadata.toZNRecord());
    final ExternalView offlineExternalView = new ExternalView("myTable_OFFLINE");
    offlineExternalView.setState("someSegment_0", "Server_1.2.3.4_1234", "ONLINE");
    final MutableBoolean timeBoundaryUpdated = new MutableBoolean(false);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(propertyStore, NO_LLC_ROUTING, null, new BaseConfiguration()) {

        @Override
        protected ExternalView fetchExternalView(String table) {
            return offlineExternalView;
        }

        @Override
        protected void updateTimeBoundary(String tableName, ExternalView externalView) {
            if (tableName.equals("myTable_OFFLINE")) {
                timeBoundaryUpdated.setValue(true);
            }
        }
    };
    routingTable.setBrokerMetrics(new BrokerMetrics(new MetricsRegistry()));
    Assert.assertFalse(timeBoundaryUpdated.booleanValue());
    final ArrayList<InstanceConfig> instanceConfigList = new ArrayList<>();
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.4_1234"));
    routingTable.markDataResourceOnline("myTable_OFFLINE", offlineExternalView, instanceConfigList);
    routingTable.markDataResourceOnline("myTable_REALTIME", new ExternalView("myTable_REALTIME"), null);
    Assert.assertTrue(timeBoundaryUpdated.booleanValue());
}

Example 17 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class HelixExternalViewBasedTimeBoundaryService method updateTimeBoundaryService.

public synchronized void updateTimeBoundaryService(ExternalView externalView) {
    if (_propertyStore == null) {
        return;
    }
    String tableName = externalView.getResourceName();
    // Do nothing for realtime table.
    if (TableNameBuilder.getTableTypeFromTableName(tableName) == TableType.REALTIME) {
        return;
    }
    Set<String> offlineSegmentsServing = externalView.getPartitionSet();
    if (offlineSegmentsServing.isEmpty()) {
        LOGGER.info("Skipping updating time boundary service for table '{}' with no offline segments.", tableName);
        return;
    }
    AbstractTableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName);
    String timeType = offlineTableConfig.getValidationConfig().getTimeType();
    TimeUnit tableTimeUnit = getTimeUnitFromString(timeType);
    if (tableTimeUnit == null) {
        LOGGER.info("Skipping updating time boundary service for table '{}' with null timeUnit, config time type: {}.", tableName, timeType);
        return;
    }
    // Bulk reading all segment zk-metadata at once is more efficient than reading one at a time.
    List<OfflineSegmentZKMetadata> segmentZKMetadataList = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(_propertyStore, tableName);
    long maxTimeValue = computeMaxSegmentEndTimeForTable(segmentZKMetadataList, tableTimeUnit);
    TimeBoundaryInfo timeBoundaryInfo = new TimeBoundaryInfo();
    timeBoundaryInfo.setTimeColumn(offlineTableConfig.getValidationConfig().getTimeColumnName());
    timeBoundaryInfo.setTimeValue(Long.toString(maxTimeValue));
    _timeBoundaryInfoMap.put(tableName, timeBoundaryInfo);
    LOGGER.info("Updated time boundary service for table '{}', maxTime: {}", tableName, maxTimeValue);
}

Example 18 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class HelixExternalViewBasedTimeBoundaryService method computeMaxSegmentEndTimeForTable.

/**
   * Compute maximum end time across a list of segment zk-metadata.
   *
   * @param segmentZKMetadataList List of Segment zk metadata for which to compute the max end time.
   * @param tableTimeUnit Time Unit for table
   * @return Max end time across all segments.
   */
private long computeMaxSegmentEndTimeForTable(List<OfflineSegmentZKMetadata> segmentZKMetadataList, TimeUnit tableTimeUnit) {
    long maxTimeValue = -1;
    for (OfflineSegmentZKMetadata metadata : segmentZKMetadataList) {
        long endTime = metadata.getEndTime();
        if (endTime <= 0) {
            continue;
        }
        // Convert all segment times to table's time unit, before comparison.
        TimeUnit segmentTimeUnit = metadata.getTimeUnit();
        if (segmentTimeUnit != null) {
            endTime = tableTimeUnit.convert(endTime, segmentTimeUnit);
        }
        maxTimeValue = Math.max(maxTimeValue, endTime);
    }
    return maxTimeValue;
}

Example 19 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class ValidationManagerTest method testPushTimePersistence.

@Test
public void testPushTimePersistence() throws Exception {
    DummyMetadata metadata = new DummyMetadata(TEST_TABLE_NAME);
    _pinotHelixResourceManager.addSegment(metadata, "http://dummy/");
    Thread.sleep(1000);
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), metadata.getTableName(), metadata.getName());
    SegmentMetadata fetchedMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
    long pushTime = fetchedMetadata.getPushTime();
    // Check that the segment has been pushed in the last 30 seconds
    Assert.assertTrue(System.currentTimeMillis() - pushTime < 30000);
    // Check that there is no refresh time
    Assert.assertEquals(fetchedMetadata.getRefreshTime(), Long.MIN_VALUE);
    // Refresh the segment
    metadata.setCrc("anotherfakecrc");
    _pinotHelixResourceManager.addSegment(metadata, "http://dummy/");
    Thread.sleep(1000);
    offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), metadata.getTableName(), metadata.getName());
    fetchedMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
    // Check that the segment still has the same push time
    Assert.assertEquals(fetchedMetadata.getPushTime(), pushTime);
    // Check that the refresh time is in the last 30 seconds
    Assert.assertTrue(System.currentTimeMillis() - fetchedMetadata.getRefreshTime() < 30000);
}