Examples with FileBasedStreamProviderImpl com.linkedin.pinot.core.realtime.impl.FileBasedStreamProviderImpl used on opensource projects

Example 1 with FileBasedStreamProviderImpl

use of com.linkedin.pinot.core.realtime.impl.FileBasedStreamProviderImpl in project pinot by linkedin.

the class RealtimeFileBasedReaderTest method setUp.

private void setUp(SegmentVersion segmentVersion) throws Exception {
    filePath = RealtimeFileBasedReaderTest.class.getClassLoader().getResource(AVRO_DATA).getFile();
    fieldTypeMap = new HashMap<>();
    fieldTypeMap.put("column1", FieldType.DIMENSION);
    fieldTypeMap.put("column2", FieldType.DIMENSION);
    fieldTypeMap.put("column3", FieldType.DIMENSION);
    fieldTypeMap.put("column4", FieldType.DIMENSION);
    fieldTypeMap.put("column5", FieldType.DIMENSION);
    fieldTypeMap.put("column6", FieldType.DIMENSION);
    fieldTypeMap.put("column7", FieldType.DIMENSION);
    fieldTypeMap.put("column8", FieldType.DIMENSION);
    fieldTypeMap.put("column9", FieldType.DIMENSION);
    fieldTypeMap.put("column10", FieldType.DIMENSION);
    fieldTypeMap.put("weeksSinceEpochSunday", FieldType.DIMENSION);
    fieldTypeMap.put("daysSinceEpoch", FieldType.DIMENSION);
    fieldTypeMap.put("column13", FieldType.TIME);
    fieldTypeMap.put("count", FieldType.METRIC);
    schema = SegmentTestUtils.extractSchemaFromAvro(new File(filePath), fieldTypeMap, TimeUnit.MINUTES);
    StreamProviderConfig config = new FileBasedStreamProviderConfig(FileFormat.AVRO, filePath, schema);
    StreamProvider provider = new FileBasedStreamProviderImpl();
    final String tableName = RealtimeFileBasedReaderTest.class.getSimpleName() + ".noTable";
    provider.init(config, tableName, new ServerMetrics(new MetricsRegistry()));
    realtimeSegment = RealtimeSegmentImplTest.createRealtimeSegmentImpl(schema, 100000, tableName, segmentName, AVRO_DATA, new ServerMetrics(new MetricsRegistry()));
    GenericRow row = provider.next(new GenericRow());
    while (row != null) {
        realtimeSegment.index(row);
        row = provider.next(row);
    }
    provider.shutdown();
    if (new File("/tmp/realtime").exists()) {
        FileUtils.deleteQuietly(new File("/tmp/realtime"));
    }
    RealtimeSegmentConverter conveter = new RealtimeSegmentConverter(realtimeSegment, "/tmp/realtime", schema, tableName, segmentName, null);
    conveter.build(segmentVersion);
    offlineSegment = Loaders.IndexSegment.load(new File("/tmp/realtime").listFiles()[0], ReadMode.mmap);
}

Example 2 with FileBasedStreamProviderImpl

use of com.linkedin.pinot.core.realtime.impl.FileBasedStreamProviderImpl in project pinot by linkedin.

the class RealtimeSegmentTest method before.

@BeforeClass
public static void before() throws Exception {
    filePath = RealtimeFileBasedReaderTest.class.getClassLoader().getResource(AVRO_DATA).getFile();
    fieldTypeMap = new HashMap<String, FieldSpec.FieldType>();
    fieldTypeMap.put("column1", FieldType.DIMENSION);
    fieldTypeMap.put("column2", FieldType.DIMENSION);
    fieldTypeMap.put("column3", FieldType.DIMENSION);
    fieldTypeMap.put("column4", FieldType.DIMENSION);
    fieldTypeMap.put("column5", FieldType.DIMENSION);
    fieldTypeMap.put("column6", FieldType.DIMENSION);
    fieldTypeMap.put("column7", FieldType.DIMENSION);
    fieldTypeMap.put("column8", FieldType.DIMENSION);
    fieldTypeMap.put("column9", FieldType.DIMENSION);
    fieldTypeMap.put("column10", FieldType.DIMENSION);
    fieldTypeMap.put("weeksSinceEpochSunday", FieldType.DIMENSION);
    fieldTypeMap.put("daysSinceEpoch", FieldType.DIMENSION);
    fieldTypeMap.put("column13", FieldType.TIME);
    fieldTypeMap.put("count", FieldType.METRIC);
    schema = SegmentTestUtils.extractSchemaFromAvro(new File(filePath), fieldTypeMap, TimeUnit.MINUTES);
    StreamProviderConfig config = new FileBasedStreamProviderConfig(FileFormat.AVRO, filePath, schema);
    //    System.out.println(config);
    StreamProvider provider = new FileBasedStreamProviderImpl();
    final String tableName = RealtimeSegmentTest.class.getSimpleName() + ".noTable";
    provider.init(config, tableName, new ServerMetrics(new MetricsRegistry()));
    List<String> invertedIdxCols = new ArrayList<>();
    invertedIdxCols.add("count");
    segmentWithInvIdx = new RealtimeSegmentImpl(schema, 100000, tableName, "noSegment", AVRO_DATA, new ServerMetrics(new MetricsRegistry()), invertedIdxCols, 2);
    segmentWithoutInvIdx = RealtimeSegmentImplTest.createRealtimeSegmentImpl(schema, 100000, tableName, "noSegment", AVRO_DATA, new ServerMetrics(new MetricsRegistry()));
    GenericRow row = provider.next(new GenericRow());
    while (row != null) {
        segmentWithInvIdx.index(row);
        segmentWithoutInvIdx.index(row);
        row = GenericRow.createOrReuseRow(row);
        row = provider.next(row);
    }
    provider.shutdown();
}