Examples with RealtimeSegmentConverter com.linkedin.pinot.core.realtime.converter.RealtimeSegmentConverter used on opensource projects

Example 1 with RealtimeSegmentConverter

use of com.linkedin.pinot.core.realtime.converter.RealtimeSegmentConverter in project pinot by linkedin.

the class LLRealtimeSegmentDataManager method buildSegment.

/**
   *
   * @param buildTgz true if you want the method to also build tgz file
   * @return true if all succeeds.
   */
protected boolean buildSegment(boolean buildTgz) {
    long startTimeMillis = System.currentTimeMillis();
    // Build a segment from in-memory rows.If buildTgz is true, then build the tar.gz file as well
    // TODO Use an auto-closeable object to delete temp resources.
    File tempSegmentFolder = new File(_resourceTmpDir, "tmp-" + _segmentNameStr + "-" + String.valueOf(now()));
    // lets convert the segment now
    RealtimeSegmentConverter converter = new RealtimeSegmentConverter(_realtimeSegment, tempSegmentFolder.getAbsolutePath(), _schema, _segmentZKMetadata.getTableName(), _segmentZKMetadata.getSegmentName(), _sortedColumn, _invertedIndexColumns, _noDictionaryColumns);
    logStatistics();
    segmentLogger.info("Trying to build segment");
    final long buildStartTime = now();
    try {
        converter.build(_segmentVersion);
    } catch (Exception e) {
        segmentLogger.error("Could not build segment", e);
        FileUtils.deleteQuietly(tempSegmentFolder);
        return false;
    }
    final long buildEndTime = now();
    segmentLogger.info("Successfully built segment in {} ms", (buildEndTime - buildStartTime));
    File destDir = makeSegmentDirPath();
    FileUtils.deleteQuietly(destDir);
    try {
        FileUtils.moveDirectory(tempSegmentFolder.listFiles()[0], destDir);
        if (buildTgz) {
            TarGzCompressionUtils.createTarGzOfDirectory(destDir.getAbsolutePath());
        }
    } catch (IOException e) {
        segmentLogger.error("Exception during move/tar segment", e);
        FileUtils.deleteQuietly(tempSegmentFolder);
        return false;
    }
    FileUtils.deleteQuietly(tempSegmentFolder);
    long endTimeMillis = System.currentTimeMillis();
    _serverMetrics.setValueOfTableGauge(_metricKeyName, ServerGauge.LAST_REALTIME_SEGMENT_CREATION_DURATION_SECONDS, TimeUnit.MILLISECONDS.toSeconds(endTimeMillis - startTimeMillis));
    return true;
}

Example 2 with RealtimeSegmentConverter

use of com.linkedin.pinot.core.realtime.converter.RealtimeSegmentConverter in project pinot by linkedin.

the class RealtimeFileBasedReaderTest method setUp.

private void setUp(SegmentVersion segmentVersion) throws Exception {
    filePath = RealtimeFileBasedReaderTest.class.getClassLoader().getResource(AVRO_DATA).getFile();
    fieldTypeMap = new HashMap<>();
    fieldTypeMap.put("column1", FieldType.DIMENSION);
    fieldTypeMap.put("column2", FieldType.DIMENSION);
    fieldTypeMap.put("column3", FieldType.DIMENSION);
    fieldTypeMap.put("column4", FieldType.DIMENSION);
    fieldTypeMap.put("column5", FieldType.DIMENSION);
    fieldTypeMap.put("column6", FieldType.DIMENSION);
    fieldTypeMap.put("column7", FieldType.DIMENSION);
    fieldTypeMap.put("column8", FieldType.DIMENSION);
    fieldTypeMap.put("column9", FieldType.DIMENSION);
    fieldTypeMap.put("column10", FieldType.DIMENSION);
    fieldTypeMap.put("weeksSinceEpochSunday", FieldType.DIMENSION);
    fieldTypeMap.put("daysSinceEpoch", FieldType.DIMENSION);
    fieldTypeMap.put("column13", FieldType.TIME);
    fieldTypeMap.put("count", FieldType.METRIC);
    schema = SegmentTestUtils.extractSchemaFromAvro(new File(filePath), fieldTypeMap, TimeUnit.MINUTES);
    StreamProviderConfig config = new FileBasedStreamProviderConfig(FileFormat.AVRO, filePath, schema);
    StreamProvider provider = new FileBasedStreamProviderImpl();
    final String tableName = RealtimeFileBasedReaderTest.class.getSimpleName() + ".noTable";
    provider.init(config, tableName, new ServerMetrics(new MetricsRegistry()));
    realtimeSegment = RealtimeSegmentImplTest.createRealtimeSegmentImpl(schema, 100000, tableName, segmentName, AVRO_DATA, new ServerMetrics(new MetricsRegistry()));
    GenericRow row = provider.next(new GenericRow());
    while (row != null) {
        realtimeSegment.index(row);
        row = provider.next(row);
    }
    provider.shutdown();
    if (new File("/tmp/realtime").exists()) {
        FileUtils.deleteQuietly(new File("/tmp/realtime"));
    }
    RealtimeSegmentConverter conveter = new RealtimeSegmentConverter(realtimeSegment, "/tmp/realtime", schema, tableName, segmentName, null);
    conveter.build(segmentVersion);
    offlineSegment = Loaders.IndexSegment.load(new File("/tmp/realtime").listFiles()[0], ReadMode.mmap);
}