Examples with MappedByteBuffer java.nio.MappedByteBuffer used on opensource projects

Example 6 with MappedByteBuffer

use of java.nio.MappedByteBuffer in project druid by druid-io.

the class StringDimensionMergerLegacy method writeIndexes.

@Override
public void writeIndexes(List<IntBuffer> segmentRowNumConversions, Closer closer) throws IOException {
    final SerializerUtils serializerUtils = new SerializerUtils();
    long dimStartTime = System.currentTimeMillis();
    final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
    String bmpFilename = String.format("%s.inverted", dimensionName);
    bitmapWriter = new GenericIndexedWriter<>(ioPeon, bmpFilename, bitmapSerdeFactory.getObjectStrategy());
    bitmapWriter.open();
    final MappedByteBuffer dimValsMapped = Files.map(dictionaryFile);
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            ByteBufferUtils.unmap(dimValsMapped);
        }
    });
    if (!dimensionName.equals(serializerUtils.readString(dimValsMapped))) {
        throw new ISE("dimensions[%s] didn't equate!?  This is a major WTF moment.", dimensionName);
    }
    Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
    log.info("Starting dimension[%s] with cardinality[%,d]", dimensionName, dimVals.size());
    final BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory();
    RTree tree = null;
    spatialWriter = null;
    boolean hasSpatial = capabilities.hasSpatialIndexes();
    if (hasSpatial) {
        String spatialFilename = String.format("%s.spatial", dimensionName);
        spatialWriter = new ByteBufferWriter<>(ioPeon, spatialFilename, new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory));
        spatialWriter.open();
        tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory);
    }
    IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
    //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result.
    for (int dictId = 0; dictId < dimVals.size(); dictId++) {
        progress.progress();
        mergeBitmaps(segmentRowNumConversions, dimVals, bmpFactory, tree, hasSpatial, dictIdSeeker, dictId, adapters, dimensionName, nullRowsBitmap, bitmapWriter);
    }
    log.info("Completed dimension[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime);
    if (hasSpatial) {
        spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
    }
}

Example 7 with MappedByteBuffer

use of java.nio.MappedByteBuffer in project druid by druid-io.

the class HadoopConverterJobTest method corrupt.

private static void corrupt(DataSegment segment) throws IOException {
    final Map<String, Object> localLoadSpec = segment.getLoadSpec();
    final Path segmentPath = Paths.get(localLoadSpec.get("path").toString());
    final MappedByteBuffer buffer = Files.map(segmentPath.toFile(), FileChannel.MapMode.READ_WRITE);
    while (buffer.hasRemaining()) {
        buffer.put((byte) 0xFF);
    }
}

Example 8 with MappedByteBuffer

use of java.nio.MappedByteBuffer in project buck by facebook.

the class AbstractElfDynamicSectionScrubberStep method execute.

@Override
public StepExecutionResult execute(ExecutionContext context) throws IOException {
    try (FileChannel channel = FileChannel.open(getFilesystem().resolve(getPath()), StandardOpenOption.READ, StandardOpenOption.WRITE)) {
        MappedByteBuffer buffer = channel.map(READ_WRITE, 0, channel.size());
        Elf elf = new Elf(buffer);
        Optional<ElfSection> section = elf.getSectionByName(SECTION).map(Pair::getSecond);
        if (!section.isPresent()) {
            throw new IOException(String.format("Error parsing ELF file %s: no such section \"%s\"", getPath(), SECTION));
        }
        for (ByteBuffer body = section.get().body; body.hasRemaining(); ) {
            ElfDynamicSection.DTag dTag = ElfDynamicSection.DTag.valueOf(elf.header.ei_class == ElfHeader.EIClass.ELFCLASS32 ? Elf.Elf32.getElf32Sword(body) : (int) Elf.Elf64.getElf64Sxword(body));
            if (!WHITELISTED_TAGS.contains(dTag)) {
                if (elf.header.ei_class == ElfHeader.EIClass.ELFCLASS32) {
                    // d_ptr
                    Elf.Elf32.putElf32Addr(body, 0);
                } else {
                    // d_ptr
                    Elf.Elf64.putElf64Addr(body, 0);
                }
            } else {
                if (elf.header.ei_class == ElfHeader.EIClass.ELFCLASS32) {
                    // d_ptr
                    Elf.Elf32.getElf32Addr(body);
                } else {
                    // d_ptr
                    Elf.Elf64.getElf64Addr(body);
                }
            }
        }
    }
    return StepExecutionResult.SUCCESS;
}

Example 9 with MappedByteBuffer

use of java.nio.MappedByteBuffer in project buck by facebook.

the class AbstractElfExtractSectionsStep method getNewSectionAddresses.

// We want to compact the sections into the new ELF file, so find out the new addresses of each
// section.
private ImmutableMap<String, Long> getNewSectionAddresses() throws IOException {
    ImmutableMap.Builder<String, Long> addresses = ImmutableMap.builder();
    try (FileChannel channel = FileChannel.open(getFilesystem().resolve(getInput()), StandardOpenOption.READ)) {
        MappedByteBuffer buffer = channel.map(READ_ONLY, 0, channel.size());
        Elf elf = new Elf(buffer);
        // We start placing sections right after the program headers.
        long end = elf.header.e_phoff + elf.header.e_phnum * elf.header.e_phentsize;
        for (int index = 0; index < elf.getNumberOfSections(); index++) {
            ElfSection section = elf.getSectionByIndex(index);
            String name = elf.getSectionName(section.header);
            // address by this sections size.
            if (getSections().contains(name)) {
                addresses.put(name, end);
                end += section.header.sh_size;
            }
        }
    }
    return addresses.build();
}

Example 10 with MappedByteBuffer

use of java.nio.MappedByteBuffer in project buck by facebook.

the class AbstractElfSymbolTableScrubberStep method execute.

@Override
public StepExecutionResult execute(ExecutionContext context) throws IOException {
    try (FileChannel channel = FileChannel.open(getFilesystem().resolve(getPath()), StandardOpenOption.READ, StandardOpenOption.WRITE)) {
        MappedByteBuffer buffer = channel.map(READ_WRITE, 0, channel.size());
        Elf elf = new Elf(buffer);
        // Locate the symbol table section.
        Optional<ElfSection> section = elf.getSectionByName(getSection()).map(Pair::getSecond);
        if (!section.isPresent()) {
            if (isAllowMissing()) {
                return StepExecutionResult.SUCCESS;
            } else {
                throw new IOException(String.format("Error parsing ELF file %s: no such section \"%s\"", getPath(), getSection()));
            }
        }
        // Read in and fixup the symbol table then write it back out.
        ElfSymbolTable table = ElfSymbolTable.parse(elf.header.ei_class, section.get().body);
        ElfSymbolTable fixedUpTable = fixUpSymbolTable(table);
        Preconditions.checkState(table.entries.size() == fixedUpTable.entries.size());
        section.get().body.rewind();
        fixedUpTable.write(elf.header.ei_class, section.get().body);
    }
    return StepExecutionResult.SUCCESS;
}