Examples with RealtimeSegmentImpl com.linkedin.pinot.core.realtime.impl.RealtimeSegmentImpl used on opensource projects

Example 1 with RealtimeSegmentImpl

use of com.linkedin.pinot.core.realtime.impl.RealtimeSegmentImpl in project pinot by linkedin.

the class LLRealtimeSegmentDataManagerTest method replaceRealtimeSegment.

// Replace the realtime segment with a mock that returns numDocs for raw doc count.
private void replaceRealtimeSegment(FakeLLRealtimeSegmentDataManager segmentDataManager, int numDocs) throws Exception {
    RealtimeSegmentImpl mockSegmentImpl = mock(RealtimeSegmentImpl.class);
    when(mockSegmentImpl.getRawDocumentCount()).thenReturn(numDocs);
    Field segmentImpl = LLRealtimeSegmentDataManager.class.getDeclaredField("_realtimeSegment");
    segmentImpl.setAccessible(true);
    segmentImpl.set(segmentDataManager, mockSegmentImpl);
}

Example 2 with RealtimeSegmentImpl

use of com.linkedin.pinot.core.realtime.impl.RealtimeSegmentImpl in project pinot by linkedin.

the class RealtimeSegmentTest method before.

@BeforeClass
public static void before() throws Exception {
    filePath = RealtimeFileBasedReaderTest.class.getClassLoader().getResource(AVRO_DATA).getFile();
    fieldTypeMap = new HashMap<String, FieldSpec.FieldType>();
    fieldTypeMap.put("column1", FieldType.DIMENSION);
    fieldTypeMap.put("column2", FieldType.DIMENSION);
    fieldTypeMap.put("column3", FieldType.DIMENSION);
    fieldTypeMap.put("column4", FieldType.DIMENSION);
    fieldTypeMap.put("column5", FieldType.DIMENSION);
    fieldTypeMap.put("column6", FieldType.DIMENSION);
    fieldTypeMap.put("column7", FieldType.DIMENSION);
    fieldTypeMap.put("column8", FieldType.DIMENSION);
    fieldTypeMap.put("column9", FieldType.DIMENSION);
    fieldTypeMap.put("column10", FieldType.DIMENSION);
    fieldTypeMap.put("weeksSinceEpochSunday", FieldType.DIMENSION);
    fieldTypeMap.put("daysSinceEpoch", FieldType.DIMENSION);
    fieldTypeMap.put("column13", FieldType.TIME);
    fieldTypeMap.put("count", FieldType.METRIC);
    schema = SegmentTestUtils.extractSchemaFromAvro(new File(filePath), fieldTypeMap, TimeUnit.MINUTES);
    StreamProviderConfig config = new FileBasedStreamProviderConfig(FileFormat.AVRO, filePath, schema);
    //    System.out.println(config);
    StreamProvider provider = new FileBasedStreamProviderImpl();
    final String tableName = RealtimeSegmentTest.class.getSimpleName() + ".noTable";
    provider.init(config, tableName, new ServerMetrics(new MetricsRegistry()));
    List<String> invertedIdxCols = new ArrayList<>();
    invertedIdxCols.add("count");
    segmentWithInvIdx = new RealtimeSegmentImpl(schema, 100000, tableName, "noSegment", AVRO_DATA, new ServerMetrics(new MetricsRegistry()), invertedIdxCols, 2);
    segmentWithoutInvIdx = RealtimeSegmentImplTest.createRealtimeSegmentImpl(schema, 100000, tableName, "noSegment", AVRO_DATA, new ServerMetrics(new MetricsRegistry()));
    GenericRow row = provider.next(new GenericRow());
    while (row != null) {
        segmentWithInvIdx.index(row);
        segmentWithoutInvIdx.index(row);
        row = GenericRow.createOrReuseRow(row);
        row = provider.next(row);
    }
    provider.shutdown();
}

Example 3 with RealtimeSegmentImpl

use of com.linkedin.pinot.core.realtime.impl.RealtimeSegmentImpl in project pinot by linkedin.

the class RealtimeQueriesSentinelTest method getRealtimeSegment.

private IndexSegment getRealtimeSegment() throws IOException {
    RealtimeSegmentImpl realtimeSegmentImpl = RealtimeSegmentImplTest.createRealtimeSegmentImpl(PINOT_SCHEMA, 100000, "testTable", "testTable_testTable", AVRO_DATA, new ServerMetrics(new MetricsRegistry()));
    realtimeSegmentImpl.setSegmentMetadata(getRealtimeSegmentZKMetadata());
    try {
        DataFileStream<GenericRecord> avroReader = AvroUtils.getAvroReader(new File(TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA))));
        GenericRow genericRow = null;
        while (avroReader.hasNext()) {
            GenericRecord avroRecord = avroReader.next();
            genericRow = GenericRow.createOrReuseRow(genericRow);
            genericRow = AVRO_RECORD_TRANSFORMER.transform(avroRecord, genericRow);
            // System.out.println(genericRow);
            realtimeSegmentImpl.index(genericRow);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    //    System.out.println("Current raw events indexed: " + realtimeSegmentImpl.getRawDocumentCount() + ", totalDocs = "
    //        + realtimeSegmentImpl.getSegmentMetadata().getTotalDocs());
    realtimeSegmentImpl.setSegmentMetadata(getRealtimeSegmentZKMetadata());
    return realtimeSegmentImpl;
}

Example 4 with RealtimeSegmentImpl

use of com.linkedin.pinot.core.realtime.impl.RealtimeSegmentImpl in project pinot by linkedin.

the class RealtimeSegmentImplTest method testDropInvalidRows.

@Test
public void testDropInvalidRows() throws Exception {
    Schema schema = new Schema.SchemaBuilder().setSchemaName("potato").addSingleValueDimension("dimension", FieldSpec.DataType.STRING).addMetric("metric", FieldSpec.DataType.LONG).addTime("time", TimeUnit.SECONDS, FieldSpec.DataType.LONG).build();
    RealtimeSegmentImpl realtimeSegment = createRealtimeSegmentImpl(schema, 100, "noTable", "noSegment", schema.getSchemaName(), new ServerMetrics(new MetricsRegistry()));
    // Segment should be empty
    Assert.assertEquals(realtimeSegment.getRawDocumentCount(), 0);
    Map<String, Object> genericRowContents = new HashMap<>();
    genericRowContents.put("dimension", "potato");
    genericRowContents.put("metric", 1234L);
    genericRowContents.put("time", 4567L);
    GenericRow row = new GenericRow();
    row.init(genericRowContents);
    // Add a valid row
    boolean notFull = realtimeSegment.index(row);
    Assert.assertEquals(notFull, true);
    Assert.assertEquals(realtimeSegment.getRawDocumentCount(), 1);
    // Add an invalid row
    genericRowContents.put("metric", null);
    notFull = realtimeSegment.index(row);
    Assert.assertEquals(notFull, true);
    Assert.assertEquals(realtimeSegment.getRawDocumentCount(), 1);
    // Add another valid row
    genericRowContents.put("metric", 2222L);
    notFull = realtimeSegment.index(row);
    Assert.assertEquals(notFull, true);
    Assert.assertEquals(realtimeSegment.getRawDocumentCount(), 2);
}