Examples with OfflineSegmentZKMetadata com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata used on opensource projects

Example 11 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class RetentionManagerTest method registerSegmentMetadata.

private void registerSegmentMetadata(SegmentMetadata segmentMetadata) {
    // put into propertyStore
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
    ZKMetadataUtils.updateSegmentMetadata(offlineSegmentZKMetadata, segmentMetadata);
    ZKMetadataProvider.setOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), offlineSegmentZKMetadata);
    // put into idealStates
    IdealState idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, _offlineTableName);
    idealState.setPartitionState(segmentMetadata.getName(), "Server_localhost_0", "ONLINE");
    idealState.setPartitionState(segmentMetadata.getName(), "Server_localhost_1", "ONLINE");
    _helixAdmin.setResourceIdealState(HELIX_CLUSTER_NAME, _offlineTableName, idealState);
}

Example 12 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class SegmentFetcherAndLoader method addOrReplaceOfflineSegment.

public void addOrReplaceOfflineSegment(String tableName, String segmentId, boolean retryOnFailure) {
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_propertyStore, tableName, segmentId);
    // Try to load table schema from Helix property store.
    // This schema is used for adding default values for newly added columns.
    Schema schema = null;
    try {
        schema = getSchema(tableName);
    } catch (Exception e) {
        LOGGER.error("Caught exception while trying to load schema for table: {}", tableName, e);
    }
    LOGGER.info("Adding or replacing segment {} for table {}, metadata {}", segmentId, tableName, offlineSegmentZKMetadata);
    try {
        SegmentMetadata segmentMetadataForCheck = new SegmentMetadataImpl(offlineSegmentZKMetadata);
        // We lock the segment in order to get its metadata, and then release the lock, so it is possible
        // that the segment is dropped after we get its metadata.
        SegmentMetadata localSegmentMetadata = _dataManager.getSegmentMetadata(tableName, segmentId);
        if (localSegmentMetadata == null) {
            LOGGER.info("Segment {} of table {} is not loaded in memory, checking disk", segmentId, tableName);
            final String localSegmentDir = getSegmentLocalDirectory(tableName, segmentId);
            if (new File(localSegmentDir).exists()) {
                LOGGER.info("Segment {} of table {} found on disk, attempting to load it", segmentId, tableName);
                try {
                    localSegmentMetadata = _metadataLoader.loadIndexSegmentMetadataFromDir(localSegmentDir);
                    LOGGER.info("Found segment {} of table {} with crc {} on disk", segmentId, tableName, localSegmentMetadata.getCrc());
                } catch (Exception e) {
                    // The localSegmentDir should help us get the table name,
                    LOGGER.error("Failed to load segment metadata from {}. Deleting it.", localSegmentDir, e);
                    FileUtils.deleteQuietly(new File(localSegmentDir));
                    localSegmentMetadata = null;
                }
                try {
                    if (!isNewSegmentMetadata(localSegmentMetadata, segmentMetadataForCheck, segmentId, tableName)) {
                        LOGGER.info("Segment metadata same as before, loading {} of table {} (crc {}) from disk", segmentId, tableName, localSegmentMetadata.getCrc());
                        AbstractTableConfig tableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName);
                        _dataManager.addSegment(localSegmentMetadata, tableConfig, schema);
                        // TODO Update zk metadata with CRC for this instance
                        return;
                    }
                } catch (V3RemoveIndexException e) {
                    LOGGER.info("Unable to remove local index from V3 format segment: {}, table: {}, try to reload it from controller.", segmentId, tableName, e);
                    FileUtils.deleteQuietly(new File(localSegmentDir));
                    localSegmentMetadata = null;
                } catch (Exception e) {
                    LOGGER.error("Failed to load {} of table {} from local, will try to reload it from controller!", segmentId, tableName, e);
                    FileUtils.deleteQuietly(new File(localSegmentDir));
                    localSegmentMetadata = null;
                }
            }
        }
        // that we have is different from that in zookeeper.
        if (isNewSegmentMetadata(localSegmentMetadata, segmentMetadataForCheck, segmentId, tableName)) {
            if (localSegmentMetadata == null) {
                LOGGER.info("Loading new segment {} of table {} from controller", segmentId, tableName);
            } else {
                LOGGER.info("Trying to refresh segment {} of table {} with new data.", segmentId, tableName);
            }
            int retryCount;
            int maxRetryCount = 1;
            if (retryOnFailure) {
                maxRetryCount = _segmentLoadMaxRetryCount;
            }
            for (retryCount = 0; retryCount < maxRetryCount; ++retryCount) {
                long attemptStartTime = System.currentTimeMillis();
                try {
                    AbstractTableConfig tableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName);
                    final String uri = offlineSegmentZKMetadata.getDownloadUrl();
                    final String localSegmentDir = downloadSegmentToLocal(uri, tableName, segmentId);
                    final SegmentMetadata segmentMetadata = _metadataLoader.loadIndexSegmentMetadataFromDir(localSegmentDir);
                    _dataManager.addSegment(segmentMetadata, tableConfig, schema);
                    LOGGER.info("Downloaded segment {} of table {} crc {} from controller", segmentId, tableName, segmentMetadata.getCrc());
                    // Successfully loaded the segment, break out of the retry loop
                    break;
                } catch (Exception e) {
                    long attemptDurationMillis = System.currentTimeMillis() - attemptStartTime;
                    LOGGER.warn("Caught exception while loading segment " + segmentId + "(table " + tableName + "), attempt " + (retryCount + 1) + " of " + maxRetryCount, e);
                    // Do we need to wait for the next retry attempt?
                    if (retryCount < maxRetryCount - 1) {
                        // Exponentially back off, wait for (minDuration + attemptDurationMillis) *
                        // 1.0..(2^retryCount)+1.0
                        double maxRetryDurationMultiplier = Math.pow(2.0, (retryCount + 1));
                        double retryDurationMultiplier = Math.random() * maxRetryDurationMultiplier + 1.0;
                        long waitTime = (long) ((_segmentLoadMinRetryDelayMs + attemptDurationMillis) * retryDurationMultiplier);
                        LOGGER.warn("Waiting for " + TimeUnit.MILLISECONDS.toSeconds(waitTime) + " seconds to retry(" + segmentId + " of table " + tableName);
                        long waitEndTime = System.currentTimeMillis() + waitTime;
                        while (System.currentTimeMillis() < waitEndTime) {
                            try {
                                Thread.sleep(Math.max(System.currentTimeMillis() - waitEndTime, 1L));
                            } catch (InterruptedException ie) {
                            // Ignore spurious wakeup
                            }
                        }
                    }
                }
            }
            if (_segmentLoadMaxRetryCount <= retryCount) {
                String msg = "Failed to download segment " + segmentId + " (table " + tableName + " after " + retryCount + " retries";
                LOGGER.error(msg);
                throw new RuntimeException(msg);
            }
        } else {
            LOGGER.info("Got already loaded segment {} of table {} crc {} again, will do nothing.", segmentId, tableName, localSegmentMetadata.getCrc());
        }
    } catch (final Exception e) {
        LOGGER.error("Cannot load segment : " + segmentId + " for table " + tableName, e);
        Utils.rethrowException(e);
        throw new AssertionError("Should not reach this");
    }
}

Example 13 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class TimeBoundaryServiceTest method constructExternalView.

private ExternalView constructExternalView(String tableName) {
    ExternalView externalView = new ExternalView(tableName);
    List<OfflineSegmentZKMetadata> offlineResourceZKMetadataListForResource = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(_propertyStore, tableName);
    for (OfflineSegmentZKMetadata segmentMetadata : offlineResourceZKMetadataListForResource) {
        externalView.setState(segmentMetadata.getSegmentName(), "localhost", "ONLINE");
    }
    return externalView;
}

Example 14 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class TimeBoundaryServiceTest method addingSegmentsToPropertyStore.

private void addingSegmentsToPropertyStore(int numSegments, ZkHelixPropertyStore<ZNRecord> propertyStore, String tableName) {
    for (int i = 0; i < numSegments; ++i) {
        OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
        offlineSegmentZKMetadata.setSegmentName(tableName + "_" + System.currentTimeMillis() + "_" + i);
        offlineSegmentZKMetadata.setTimeUnit(TimeUnit.DAYS);
        offlineSegmentZKMetadata.setEndTime(i);
        offlineSegmentZKMetadata.setCrc(-1);
        offlineSegmentZKMetadata.setCreationTime(-1);
        offlineSegmentZKMetadata.setStartTime(i - 1);
        offlineSegmentZKMetadata.setIndexVersion("0");
        offlineSegmentZKMetadata.setPushTime(i + 5);
        offlineSegmentZKMetadata.setTableName(tableName);
        offlineSegmentZKMetadata.setSegmentType(SegmentType.OFFLINE);
        ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata);
    }
}

Example 15 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class SegmentZKMetadataTest method offlineSegmentZKMetadataConvertionTest.

@Test
public void offlineSegmentZKMetadataConvertionTest() {
    ZNRecord offlineZNRecord = getTestOfflineSegmentZNRecord();
    OfflineSegmentZKMetadata offlineSegmentMetadata = getTestOfflineSegmentMetadata();
    Assert.assertTrue(MetadataUtils.comparisonZNRecords(offlineZNRecord, offlineSegmentMetadata.toZNRecord()));
    Assert.assertTrue(offlineSegmentMetadata.equals(new OfflineSegmentZKMetadata(offlineZNRecord)));
    Assert.assertTrue(MetadataUtils.comparisonZNRecords(offlineZNRecord, new OfflineSegmentZKMetadata(offlineZNRecord).toZNRecord()));
    Assert.assertTrue(offlineSegmentMetadata.equals(new OfflineSegmentZKMetadata(offlineSegmentMetadata.toZNRecord())));
}