Examples with SegmentMetadataImpl com.linkedin.pinot.core.segment.index.SegmentMetadataImpl used on opensource projects

Example 16 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class SegmentFetcherAndLoader method addOrReplaceOfflineSegment.

public void addOrReplaceOfflineSegment(String tableName, String segmentId, boolean retryOnFailure) {
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_propertyStore, tableName, segmentId);
    // Try to load table schema from Helix property store.
    // This schema is used for adding default values for newly added columns.
    Schema schema = null;
    try {
        schema = getSchema(tableName);
    } catch (Exception e) {
        LOGGER.error("Caught exception while trying to load schema for table: {}", tableName, e);
    }
    LOGGER.info("Adding or replacing segment {} for table {}, metadata {}", segmentId, tableName, offlineSegmentZKMetadata);
    try {
        SegmentMetadata segmentMetadataForCheck = new SegmentMetadataImpl(offlineSegmentZKMetadata);
        // We lock the segment in order to get its metadata, and then release the lock, so it is possible
        // that the segment is dropped after we get its metadata.
        SegmentMetadata localSegmentMetadata = _dataManager.getSegmentMetadata(tableName, segmentId);
        if (localSegmentMetadata == null) {
            LOGGER.info("Segment {} of table {} is not loaded in memory, checking disk", segmentId, tableName);
            final String localSegmentDir = getSegmentLocalDirectory(tableName, segmentId);
            if (new File(localSegmentDir).exists()) {
                LOGGER.info("Segment {} of table {} found on disk, attempting to load it", segmentId, tableName);
                try {
                    localSegmentMetadata = _metadataLoader.loadIndexSegmentMetadataFromDir(localSegmentDir);
                    LOGGER.info("Found segment {} of table {} with crc {} on disk", segmentId, tableName, localSegmentMetadata.getCrc());
                } catch (Exception e) {
                    // The localSegmentDir should help us get the table name,
                    LOGGER.error("Failed to load segment metadata from {}. Deleting it.", localSegmentDir, e);
                    FileUtils.deleteQuietly(new File(localSegmentDir));
                    localSegmentMetadata = null;
                }
                try {
                    if (!isNewSegmentMetadata(localSegmentMetadata, segmentMetadataForCheck, segmentId, tableName)) {
                        LOGGER.info("Segment metadata same as before, loading {} of table {} (crc {}) from disk", segmentId, tableName, localSegmentMetadata.getCrc());
                        AbstractTableConfig tableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName);
                        _dataManager.addSegment(localSegmentMetadata, tableConfig, schema);
                        // TODO Update zk metadata with CRC for this instance
                        return;
                    }
                } catch (V3RemoveIndexException e) {
                    LOGGER.info("Unable to remove local index from V3 format segment: {}, table: {}, try to reload it from controller.", segmentId, tableName, e);
                    FileUtils.deleteQuietly(new File(localSegmentDir));
                    localSegmentMetadata = null;
                } catch (Exception e) {
                    LOGGER.error("Failed to load {} of table {} from local, will try to reload it from controller!", segmentId, tableName, e);
                    FileUtils.deleteQuietly(new File(localSegmentDir));
                    localSegmentMetadata = null;
                }
            }
        }
        // that we have is different from that in zookeeper.
        if (isNewSegmentMetadata(localSegmentMetadata, segmentMetadataForCheck, segmentId, tableName)) {
            if (localSegmentMetadata == null) {
                LOGGER.info("Loading new segment {} of table {} from controller", segmentId, tableName);
            } else {
                LOGGER.info("Trying to refresh segment {} of table {} with new data.", segmentId, tableName);
            }
            int retryCount;
            int maxRetryCount = 1;
            if (retryOnFailure) {
                maxRetryCount = _segmentLoadMaxRetryCount;
            }
            for (retryCount = 0; retryCount < maxRetryCount; ++retryCount) {
                long attemptStartTime = System.currentTimeMillis();
                try {
                    AbstractTableConfig tableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName);
                    final String uri = offlineSegmentZKMetadata.getDownloadUrl();
                    final String localSegmentDir = downloadSegmentToLocal(uri, tableName, segmentId);
                    final SegmentMetadata segmentMetadata = _metadataLoader.loadIndexSegmentMetadataFromDir(localSegmentDir);
                    _dataManager.addSegment(segmentMetadata, tableConfig, schema);
                    LOGGER.info("Downloaded segment {} of table {} crc {} from controller", segmentId, tableName, segmentMetadata.getCrc());
                    // Successfully loaded the segment, break out of the retry loop
                    break;
                } catch (Exception e) {
                    long attemptDurationMillis = System.currentTimeMillis() - attemptStartTime;
                    LOGGER.warn("Caught exception while loading segment " + segmentId + "(table " + tableName + "), attempt " + (retryCount + 1) + " of " + maxRetryCount, e);
                    // Do we need to wait for the next retry attempt?
                    if (retryCount < maxRetryCount - 1) {
                        // Exponentially back off, wait for (minDuration + attemptDurationMillis) *
                        // 1.0..(2^retryCount)+1.0
                        double maxRetryDurationMultiplier = Math.pow(2.0, (retryCount + 1));
                        double retryDurationMultiplier = Math.random() * maxRetryDurationMultiplier + 1.0;
                        long waitTime = (long) ((_segmentLoadMinRetryDelayMs + attemptDurationMillis) * retryDurationMultiplier);
                        LOGGER.warn("Waiting for " + TimeUnit.MILLISECONDS.toSeconds(waitTime) + " seconds to retry(" + segmentId + " of table " + tableName);
                        long waitEndTime = System.currentTimeMillis() + waitTime;
                        while (System.currentTimeMillis() < waitEndTime) {
                            try {
                                Thread.sleep(Math.max(System.currentTimeMillis() - waitEndTime, 1L));
                            } catch (InterruptedException ie) {
                            // Ignore spurious wakeup
                            }
                        }
                    }
                }
            }
            if (_segmentLoadMaxRetryCount <= retryCount) {
                String msg = "Failed to download segment " + segmentId + " (table " + tableName + " after " + retryCount + " retries";
                LOGGER.error(msg);
                throw new RuntimeException(msg);
            }
        } else {
            LOGGER.info("Got already loaded segment {} of table {} crc {} again, will do nothing.", segmentId, tableName, localSegmentMetadata.getCrc());
        }
    } catch (final Exception e) {
        LOGGER.error("Cannot load segment : " + segmentId + " for table " + tableName, e);
        Utils.rethrowException(e);
        throw new AssertionError("Should not reach this");
    }
}

Example 17 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class PinotSegmentUploadRestletResource method uploadSegment.

@HttpVerb("post")
@Summary("Uploads a segment")
@Tags({ "segment" })
@Paths({ "/segments", "/segments/" })
@Responses({ @Response(statusCode = "200", description = "The segment was successfully uploaded"), @Response(statusCode = "403", description = "Forbidden operation typically because it exceeds configured quota"), @Response(statusCode = "500", description = "There was an error when uploading the segment") })
private Representation uploadSegment(File indexDir, File dataFile, String downloadUrl) throws ConfigurationException, IOException, JSONException {
    final SegmentMetadata metadata = new SegmentMetadataImpl(indexDir);
    final File tableDir = new File(baseDataDir, metadata.getTableName());
    String tableName = metadata.getTableName();
    File segmentFile = new File(tableDir, dataFile.getName());
    OfflineTableConfig offlineTableConfig = (OfflineTableConfig) ZKMetadataProvider.getOfflineTableConfig(_pinotHelixResourceManager.getPropertyStore(), tableName);
    if (offlineTableConfig == null) {
        LOGGER.info("Missing configuration for table: {} in helix", metadata.getTableName());
        setStatus(Status.CLIENT_ERROR_NOT_FOUND);
        StringRepresentation repr = new StringRepresentation("{\"error\" : \"Missing table: " + tableName + "\"}");
        repr.setMediaType(MediaType.APPLICATION_JSON);
        return repr;
    }
    StorageQuotaChecker.QuotaCheckerResponse quotaResponse = checkStorageQuota(indexDir, metadata, offlineTableConfig);
    if (!quotaResponse.isSegmentWithinQuota) {
        // this is not an "error" hence we don't increment segment upload errors
        LOGGER.info("Rejecting segment upload for table: {}, segment: {}, reason: {}", metadata.getTableName(), metadata.getName(), quotaResponse.reason);
        setStatus(Status.CLIENT_ERROR_FORBIDDEN);
        StringRepresentation repr = new StringRepresentation("{\"error\" : \"" + quotaResponse.reason + "\"}");
        repr.setMediaType(MediaType.APPLICATION_JSON);
        return repr;
    }
    PinotResourceManagerResponse response;
    if (!isSegmentTimeValid(metadata)) {
        response = new PinotResourceManagerResponse("Invalid segment start/end time", false);
    } else {
        if (downloadUrl == null) {
            // serve the data downloading from Pinot Servers.
            if (segmentFile.exists()) {
                FileUtils.deleteQuietly(segmentFile);
            }
            FileUtils.moveFile(dataFile, segmentFile);
            downloadUrl = ControllerConf.constructDownloadUrl(tableName, dataFile.getName(), vip);
        }
        // TODO: this will read table configuration again from ZK. We should optimize that
        response = _pinotHelixResourceManager.addSegment(metadata, downloadUrl);
    }
    if (response.isSuccessful()) {
        setStatus(Status.SUCCESS_OK);
    } else {
        ControllerRestApplication.getControllerMetrics().addMeteredGlobalValue(ControllerMeter.CONTROLLER_SEGMENT_UPLOAD_ERROR, 1L);
        setStatus(Status.SERVER_ERROR_INTERNAL);
    }
    return new StringRepresentation(response.toJSON().toString());
}

Example 18 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class ValidationManagerTest method testPushTimePersistence.

@Test
public void testPushTimePersistence() throws Exception {
    DummyMetadata metadata = new DummyMetadata(TEST_TABLE_NAME);
    _pinotHelixResourceManager.addSegment(metadata, "http://dummy/");
    Thread.sleep(1000);
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), metadata.getTableName(), metadata.getName());
    SegmentMetadata fetchedMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
    long pushTime = fetchedMetadata.getPushTime();
    // Check that the segment has been pushed in the last 30 seconds
    Assert.assertTrue(System.currentTimeMillis() - pushTime < 30000);
    // Check that there is no refresh time
    Assert.assertEquals(fetchedMetadata.getRefreshTime(), Long.MIN_VALUE);
    // Refresh the segment
    metadata.setCrc("anotherfakecrc");
    _pinotHelixResourceManager.addSegment(metadata, "http://dummy/");
    Thread.sleep(1000);
    offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), metadata.getTableName(), metadata.getName());
    fetchedMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
    // Check that the segment still has the same push time
    Assert.assertEquals(fetchedMetadata.getPushTime(), pushTime);
    // Check that the refresh time is in the last 30 seconds
    Assert.assertTrue(System.currentTimeMillis() - fetchedMetadata.getRefreshTime() < 30000);
}

Example 19 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class SegmentV1V2ToV3FormatConverterTest method testConvert.

@Test
public void testConvert() throws Exception {
    SegmentMetadataImpl beforeConversionMeta = new SegmentMetadataImpl(segmentDirectory);
    SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
    converter.convert(segmentDirectory);
    File v3Location = SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3);
    Assert.assertTrue(v3Location.exists());
    Assert.assertTrue(v3Location.isDirectory());
    Assert.assertTrue(new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).exists());
    SegmentMetadataImpl metadata = new SegmentMetadataImpl(v3Location);
    Assert.assertEquals(metadata.getVersion(), SegmentVersion.v3.toString());
    Assert.assertTrue(new File(v3Location, V1Constants.SEGMENT_CREATION_META).exists());
    // Drop the star tree index file because it has invalid data
    new File(v3Location, V1Constants.STAR_TREE_INDEX_FILE).delete();
    new File(segmentDirectory, V1Constants.STAR_TREE_INDEX_FILE).delete();
    FileTime afterConversionTime = Files.getLastModifiedTime(v3Location.toPath());
    // verify that the segment loads correctly. This is necessary and sufficient
    // full proof way to ensure that segment is correctly translated
    IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v3LoadingConfig);
    Assert.assertNotNull(indexSegment);
    Assert.assertEquals(indexSegment.getSegmentName(), metadata.getName());
    Assert.assertEquals(SegmentVersion.v3, SegmentVersion.valueOf(indexSegment.getSegmentMetadata().getVersion()));
    FileTime afterLoadTime = Files.getLastModifiedTime(v3Location.toPath());
    Assert.assertEquals(afterConversionTime, afterLoadTime);
    // verify that SegmentMetadataImpl loaded from segmentDirectory correctly sets
    // metadata information after conversion. This has impacted us while loading
    // segments by triggering download. That's costly. That's also difficult to test
    Assert.assertFalse(new File(segmentDirectory, V1Constants.MetadataKeys.METADATA_FILE_NAME).exists());
    SegmentMetadataImpl metaAfterConversion = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertNotNull(metaAfterConversion);
    Assert.assertFalse(metaAfterConversion.getCrc().equalsIgnoreCase(String.valueOf(Long.MIN_VALUE)));
    Assert.assertEquals(metaAfterConversion.getCrc(), beforeConversionMeta.getCrc());
    Assert.assertTrue(metaAfterConversion.getIndexCreationTime() != Long.MIN_VALUE);
    Assert.assertEquals(metaAfterConversion.getIndexCreationTime(), beforeConversionMeta.getIndexCreationTime());
}

Example 20 with SegmentMetadataImpl

use of com.linkedin.pinot.core.segment.index.SegmentMetadataImpl in project pinot by linkedin.

the class LoadersTest method testLoad.

@Test
public void testLoad() throws Exception {
    SegmentMetadataImpl originalMetadata = new SegmentMetadataImpl(segmentDirectory);
    Assert.assertEquals(originalMetadata.getSegmentVersion(), SegmentVersion.v1);
    // note: ordering of these two test blocks matters
    {
        // Explicitly pass v1 format since we will convert by default to v3
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap, v1LoadingConfig);
        Assert.assertEquals(indexSegment.getSegmentMetadata().getVersion(), originalMetadata.getVersion());
        Assert.assertFalse(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    }
    {
        // with this code and converter in place, make sure we still load original version
        // by default. We require specific configuration for v3.
        IndexSegment indexSegment = Loaders.IndexSegment.load(segmentDirectory, ReadMode.mmap);
        Assert.assertEquals(indexSegment.getSegmentMetadata().getVersion(), SegmentVersion.v3.toString());
        Assert.assertTrue(SegmentDirectoryPaths.segmentDirectoryFor(segmentDirectory, SegmentVersion.v3).exists());
    }
}