Examples with LongArrayList it.unimi.dsi.fastutil.longs.LongArrayList used on opensource projects

Example 21 with LongArrayList

use of it.unimi.dsi.fastutil.longs.LongArrayList in project caffeine by ben-manes.

the class PolicyActor method onReceive.

@Override
public void onReceive(Object msg) {
    if (msg instanceof LongArrayList) {
        LongArrayList events = (LongArrayList) msg;
        process(events);
    } else if (msg == Message.FINISH) {
        policy.finished();
        getSender().tell(policy.stats(), ActorRef.noSender());
        getContext().stop(getSelf());
    } else if (msg == Message.ERROR) {
        getContext().stop(getSelf());
    } else {
        context().system().log().error("Invalid message: " + msg);
    }
}

Example 22 with LongArrayList

use of it.unimi.dsi.fastutil.longs.LongArrayList in project presto by prestodb.

the class ParquetHiveRecordCursor method createParquetRecordReader.

private ParquetRecordReader<FakeParquetRecord> createParquetRecordReader(HdfsEnvironment hdfsEnvironment, String sessionUser, Configuration configuration, Path path, long start, long length, List<HiveColumnHandle> columns, boolean useParquetColumnNames, TypeManager typeManager, boolean predicatePushdownEnabled, TupleDomain<HiveColumnHandle> effectivePredicate) {
    ParquetDataSource dataSource = null;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(sessionUser, path, configuration);
        dataSource = buildHdfsParquetDataSource(fileSystem, path, start, length);
        ParquetMetadata parquetMetadata = hdfsEnvironment.doAs(sessionUser, () -> ParquetFileReader.readFooter(configuration, path, NO_FILTER));
        List<BlockMetaData> blocks = parquetMetadata.getBlocks();
        FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
        MessageType fileSchema = fileMetaData.getSchema();
        PrestoReadSupport readSupport = new PrestoReadSupport(useParquetColumnNames, columns, fileSchema);
        List<parquet.schema.Type> fields = columns.stream().filter(column -> column.getColumnType() == REGULAR).map(column -> getParquetType(column, fileSchema, useParquetColumnNames)).filter(Objects::nonNull).collect(toList());
        MessageType requestedSchema = new MessageType(fileSchema.getName(), fields);
        LongArrayList offsets = new LongArrayList(blocks.size());
        for (BlockMetaData block : blocks) {
            long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset();
            if (firstDataPage >= start && firstDataPage < start + length) {
                if (predicatePushdownEnabled) {
                    ParquetPredicate parquetPredicate = buildParquetPredicate(columns, effectivePredicate, fileMetaData.getSchema(), typeManager);
                    if (predicateMatches(parquetPredicate, block, dataSource, requestedSchema, effectivePredicate)) {
                        offsets.add(block.getStartingPos());
                    }
                } else {
                    offsets.add(block.getStartingPos());
                }
            }
        }
        ParquetInputSplit split = new ParquetInputSplit(path, start, start + length, length, null, offsets.toLongArray());
        TaskAttemptContext taskContext = ContextUtil.newTaskAttemptContext(configuration, new TaskAttemptID());
        return hdfsEnvironment.doAs(sessionUser, () -> {
            ParquetRecordReader<FakeParquetRecord> realReader = new PrestoParquetRecordReader(readSupport);
            realReader.initialize(split, taskContext);
            return realReader;
        });
    } catch (Exception e) {
        Throwables.propagateIfInstanceOf(e, PrestoException.class);
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
            throw Throwables.propagate(e);
        }
        String message = format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, e.getMessage());
        if (e.getClass().getSimpleName().equals("BlockMissingException")) {
            throw new PrestoException(HIVE_MISSING_DATA, message, e);
        }
        throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e);
    } finally {
        if (dataSource != null) {
            try {
                dataSource.close();
            } catch (IOException ignored) {
            }
        }
    }
}

Example 23 with LongArrayList

use of it.unimi.dsi.fastutil.longs.LongArrayList in project cdap by caskdata.

the class StreamDataFileIndex method loadIndex.

private Map.Entry<LongList, LongList> loadIndex(InputStream input) throws IOException {
    byte[] magic = new byte[INDEX_MAGIC_HEADER.length];
    ByteStreams.readFully(input, magic);
    if (!Arrays.equals(magic, INDEX_MAGIC_HEADER)) {
        throw new IOException("Unsupported index file format. Expected magic bytes as 'I' '1'");
    }
    // Decode the properties map. In current version, it is not used.
    StreamUtils.decodeMap(new BinaryDecoder(input));
    // Read in all index (timestamp, position pairs).
    LongList timestamps = new LongArrayList(1000);
    LongList positions = new LongArrayList(1000);
    byte[] buf = new byte[Longs.BYTES * 2];
    while (ByteStreams.read(input, buf, 0, buf.length) == buf.length) {
        timestamps.add(Bytes.toLong(buf, 0));
        positions.add(Bytes.toLong(buf, Longs.BYTES));
    }
    return Maps.immutableEntry(timestamps, positions);
}

Example 24 with LongArrayList

use of it.unimi.dsi.fastutil.longs.LongArrayList in project mkgmap by openstreetmap.

the class HousenumberGenerator method useAddrPlaceTag.

private void useAddrPlaceTag(List<HousenumberRoad> hnrList) {
    HashMap<CityInfo, MultiHashMap<String, HousenumberMatch>> cityPlaceHouseMap = new LinkedHashMap<>();
    for (int i = 0; i < houseElems.size(); i++) {
        HousenumberElem house = houseElems.get(i);
        if (house.getRoad() == null)
            continue;
        if (house.getPlace() == null)
            continue;
        if (house instanceof HousenumberMatch) {
            HousenumberMatch hm = (HousenumberMatch) house;
            if (hm.getHousenumberRoad() == null)
                continue;
        } else
            continue;
        MultiHashMap<String, HousenumberMatch> subMap = cityPlaceHouseMap.get(house.getCityInfo());
        if (subMap == null) {
            subMap = new MultiHashMap<>();
            cityPlaceHouseMap.put(house.getCityInfo(), subMap);
        }
        subMap.add(house.getPlace(), (HousenumberMatch) house);
    }
    log.info("analysing", cityPlaceHouseMap.size(), "cities with addr:place=* houses");
    for (Entry<CityInfo, MultiHashMap<String, HousenumberMatch>> topEntry : cityPlaceHouseMap.entrySet()) {
        CityInfo cityInfo = topEntry.getKey();
        List<String> placeNames = new ArrayList<>(topEntry.getValue().keySet());
        Collections.sort(placeNames);
        for (String placeName : placeNames) {
            List<HousenumberMatch> placeHouses = topEntry.getValue().get(placeName);
            HashSet<HousenumberRoad> roads = new LinkedHashSet<>();
            Int2IntOpenHashMap usedNumbers = new Int2IntOpenHashMap();
            HashMap<String, Integer> usedSigns = new HashMap<>();
            int dupSigns = 0;
            int dupNumbers = 0;
            int housesWithStreet = 0;
            int housesWithMatchingStreet = 0;
            int roadsWithNames = 0;
            int unnamedCloseRoads = 0;
            for (HousenumberMatch house : placeHouses) {
                if (house.getStreet() != null) {
                    ++housesWithStreet;
                    if (house.getStreet().equalsIgnoreCase(house.getRoad().getStreet())) {
                        ++housesWithMatchingStreet;
                    }
                } else {
                    if (house.getRoad().getStreet() == null)
                        ++unnamedCloseRoads;
                }
                boolean added = roads.add(house.getHousenumberRoad());
                if (added && house.getRoad().getStreet() != null)
                    ++roadsWithNames;
                int oldCount = usedNumbers.put(house.getHousenumber(), 1);
                if (oldCount != 0) {
                    usedNumbers.put(house.getHousenumber(), oldCount + 1);
                    ++dupNumbers;
                }
                Integer oldSignCount = usedSigns.put(house.getSign(), 1);
                if (oldSignCount != null) {
                    usedSigns.put(house.getSign(), oldSignCount + 1);
                    ++dupSigns;
                }
            }
            if (log.isDebugEnabled()) {
                log.debug("place", placeName, "in city", cityInfo, ":", "houses:", placeHouses.size(), ",duplicate numbers/signs:", dupNumbers + "/" + dupSigns, ",roads (named/unnamed):", roads.size(), "(" + roadsWithNames + "/" + (roads.size() - roadsWithNames) + ")", ",houses without addr:street:", placeHouses.size() - housesWithStreet, ",street = name of closest road:", housesWithMatchingStreet, ",houses without addr:street near named road:", unnamedCloseRoads);
            }
            if ((float) dupSigns / placeHouses.size() < 0.25) {
                if (log.isDebugEnabled())
                    log.debug("will not use gaps in intervals for roads in", placeName);
                for (HousenumberRoad hnr : roads) {
                    hnr.setRemoveGaps(true);
                }
            }
            if (placeHouses.size() > housesWithStreet) {
                // XXX: threshold value?
                LongArrayList ids = new LongArrayList();
                for (HousenumberRoad hnr : roads) {
                    ids.add(hnr.getRoad().getRoadDef().getId());
                    hnr.addPlaceName(placeName);
                }
                if (log.isDebugEnabled())
                    log.debug("detected", placeName, "as potential address name for roads", ids);
            } else {
                if (log.isDebugEnabled())
                    log.debug("will ignore addr:place for address search in", placeName, "in city", cityInfo);
            }
        }
    }
}

Example 25 with LongArrayList

use of it.unimi.dsi.fastutil.longs.LongArrayList in project gridss by PapenfussLab.

the class KmerEncodingHelperTest method partialSequenceBasesDifferent_should_calc_forward_anchored.

@Test
public void partialSequenceBasesDifferent_should_calc_forward_anchored() {
    for (int k = 1; k < 4; k++) {
        LongArrayList ref = KPN(k, "ACTGGTTAACACGTCAGGTACGTCG", 1, 1, true).pathKmers();
        assertEquals(0, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ACTGGTTAACACGTCAGGTACGTCG", 1, 1, true).pathKmers(), 0, true));
        assertEquals(0, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ACTGGTTAACACGTCAGGTACGTCG", 1, 1, true).pathKmers(), 0, false));
        assertEquals(1, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ACTGGTTAACACGTCAGGTACTTCG", 1, 1, true).pathKmers(), 0, true));
        assertEquals(1, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ACTGGTTAACACGTCAGGTACTTCG", 1, 1, true).pathKmers(), 0, false));
        // *
        assertEquals(2, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ACTGGTTAACACGTCAGGTAGTTCG", 1, 1, true).pathKmers(), 0, true));
        assertEquals(2, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ACTGGTTAACACGTCAGGTAGTTCG", 1, 1, true).pathKmers(), 0, false));
        // **
        assertEquals(3, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ATTGGTTAACACGTCAGGTAGTTCG", 1, 1, true).pathKmers(), 0, true));
        assertEquals(3, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "ATTGGTTAACACGTCAGGTAGTTCG", 1, 1, true).pathKmers(), 0, false));
        // *                  **
        // ACTGGTTAACACGTCAGGTACGTCG
        assertEquals(0, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "CTGGTTAACACGTCAGGTACGTC", 1, 1, true).pathKmers(), 1, true));
        assertEquals(0, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "CTGGTTAACACGTCAGGTACGTC", 1, 1, true).pathKmers(), 1, false));
        // ACTG*TTAACACGTCAGGTACGTCG
        assertEquals(1, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "CTGTTTAACACGTCAGGTACGTC", 1, 1, true).pathKmers(), 1, true));
        assertEquals(1, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "CTGTTTAACACGTCAGGTACGTC", 1, 1, true).pathKmers(), 1, false));
        // ACTG*TTAACACGTCAGGTACGTCG
        assertEquals(1, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "CTGTTTAACACGTCAGGTACGTC", 1, 1, true).pathKmers(), 1, true));
        assertEquals(1, KmerEncodingHelper.partialSequenceBasesDifferent(k, ref, KPN(k, "CTGTTTAACACGTCAGGTACGTC", 1, 1, true).pathKmers(), 1, false));
    }
}