Examples with PinotDataBuffer com.linkedin.pinot.core.segment.memory.PinotDataBuffer used on opensource projects

Example 31 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FilePerIndexDirectoryTest method testEmptyDirectory.

@Test
public void testEmptyDirectory() throws Exception {
    Assert.assertEquals(0, segmentDir.list().length, segmentDir.list().toString());
    try (FilePerIndexDirectory fpiDir = new FilePerIndexDirectory(segmentDir, segmentMetadata, ReadMode.heap);
        PinotDataBuffer buffer = fpiDir.newDictionaryBuffer("col1", 1024)) {
        Assert.assertEquals(1, segmentDir.list().length, segmentDir.list().toString());
        buffer.putLong(0, 0xbadfadL);
        buffer.putInt(8, 51);
        // something at random location
        buffer.putInt(101, 55);
    }
    Assert.assertEquals(1, segmentDir.list().length);
    try (FilePerIndexDirectory colDir = new FilePerIndexDirectory(segmentDir, segmentMetadata, ReadMode.mmap);
        PinotDataBuffer readBuffer = colDir.getDictionaryBufferFor("col1")) {
        Assert.assertEquals(readBuffer.getLong(0), 0xbadfadL);
        Assert.assertEquals(readBuffer.getInt(8), 51);
        Assert.assertEquals(readBuffer.getInt(101), 55);
    }
}

Example 32 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class SegmentLocalFSDirectoryTest method testWriteAndReadBackData.

@Test
public void testWriteAndReadBackData() throws java.lang.Exception {
    try (SegmentLocalFSDirectory.Writer writer = segmentDirectory.createWriter()) {
        Assert.assertNotNull(writer);
        PinotDataBuffer buffer = writer.newIndexFor("newColumn", ColumnIndexType.FORWARD_INDEX, 1024);
        loadData(buffer);
        writer.saveAndClose();
    }
    try (SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
        Assert.assertNotNull(reader);
        PinotDataBuffer newDataBuffer = reader.getIndexFor("newColumn", ColumnIndexType.FORWARD_INDEX);
        verifyData(newDataBuffer);
    }
}

Example 33 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class SingleFileIndexDirectoryTest method testWithEmptyDir.

@Test
public void testWithEmptyDir() throws Exception {
    // segmentDir does not have anything to begin with
    Assert.assertEquals(segmentDir.list().length, 0);
    SingleFileIndexDirectory columnDirectory = new SingleFileIndexDirectory(segmentDir, segmentMetadata, ReadMode.mmap);
    PinotDataBuffer writtenBuffer = columnDirectory.newDictionaryBuffer("foo", 1024);
    String data = new String("This is a test string");
    final byte[] dataBytes = data.getBytes();
    int pos = 0;
    for (byte b : dataBytes) {
        writtenBuffer.putByte(pos++, b);
    }
    writtenBuffer.close();
    when(segmentMetadata.getAllColumns()).thenReturn(new HashSet<String>(Arrays.asList("foo")));
    try (SingleFileIndexDirectory directoryReader = new SingleFileIndexDirectory(segmentDir, segmentMetadata, ReadMode.mmap);
        PinotDataBuffer readBuffer = directoryReader.getDictionaryBufferFor("foo")) {
        Assert.assertEquals(1024, readBuffer.size());
        int length = dataBytes.length;
        for (int i = 0; i < length; i++) {
            byte b = readBuffer.getByte(i);
            Assert.assertEquals(dataBytes[i], b);
        }
    }
}

Example 34 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class SegmentPreProcessorTest method testV3CreateInvertedIndices.

@Test
public void testV3CreateInvertedIndices() throws Exception {
    constructSegment();
    // Convert segment format to v3.
    SegmentV1V2ToV3FormatConverter converter = new SegmentV1V2ToV3FormatConverter();
    converter.convert(_segmentDirectoryFile);
    File v3SegmentDirectoryFile = new File(_segmentDirectoryFile, V3_SEGMENT_NAME);
    SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(v3SegmentDirectoryFile);
    String segmentVersion = segmentMetadata.getVersion();
    Assert.assertEquals(SegmentVersion.valueOf(segmentVersion), SegmentVersion.v3);
    File singleFileIndex = new File(v3SegmentDirectoryFile, "columns.psf");
    FileTime lastModifiedTime = Files.getLastModifiedTime(singleFileIndex.toPath());
    long fileSize = singleFileIndex.length();
    // Sleep 2 seconds to prevent the same last modified time when modifying the file.
    Thread.sleep(2000);
    // Create inverted index the first time.
    checkInvertedIndexCreation(v3SegmentDirectoryFile, segmentMetadata, false);
    long addedLength = 0L;
    try (SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(v3SegmentDirectoryFile, segmentMetadata, ReadMode.mmap);
        SegmentDirectory.Reader reader = segmentDirectory.createReader()) {
        // 8 bytes overhead is for checking integrity of the segment.
        try (PinotDataBuffer col1Buffer = reader.getIndexFor(COLUMN1_NAME, ColumnIndexType.INVERTED_INDEX)) {
            addedLength += col1Buffer.size() + 8;
        }
        try (PinotDataBuffer col13Buffer = reader.getIndexFor(COLUMN13_NAME, ColumnIndexType.INVERTED_INDEX)) {
            addedLength += col13Buffer.size() + 8;
        }
    }
    FileTime newLastModifiedTime = Files.getLastModifiedTime(singleFileIndex.toPath());
    Assert.assertTrue(newLastModifiedTime.compareTo(lastModifiedTime) > 0);
    long newFileSize = singleFileIndex.length();
    Assert.assertEquals(fileSize + addedLength, newFileSize);
    // Sleep 2 seconds to prevent the same last modified time when modifying the file.
    Thread.sleep(2000);
    // Create inverted index the second time.
    checkInvertedIndexCreation(v3SegmentDirectoryFile, segmentMetadata, true);
    Assert.assertEquals(Files.getLastModifiedTime(singleFileIndex.toPath()), newLastModifiedTime);
    Assert.assertEquals(singleFileIndex.length(), newFileSize);
}

Example 35 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FilePerIndexDirectoryTest method testRemoveIndex.

@Test
public void testRemoveIndex() throws IOException {
    try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(segmentDir, segmentMetadata, ReadMode.mmap)) {
        try (PinotDataBuffer buffer = fpi.newForwardIndexBuffer("col1", 1024)) {
        }
        try (PinotDataBuffer buffer = fpi.newDictionaryBuffer("col2", 100)) {
        }
        Assert.assertTrue(fpi.getFileFor("col1", ColumnIndexType.FORWARD_INDEX).exists());
        Assert.assertTrue(fpi.getFileFor("col2", ColumnIndexType.DICTIONARY).exists());
        Assert.assertTrue(fpi.isIndexRemovalSupported());
        fpi.removeIndex("col1", ColumnIndexType.FORWARD_INDEX);
        Assert.assertFalse(fpi.getFileFor("col1", ColumnIndexType.FORWARD_INDEX).exists());
    }
}