Examples with NavigableMap java.util.NavigableMap used on opensource projects

Example 56 with NavigableMap

use of java.util.NavigableMap in project Solbase by Photobucket.

the class DocumentLoader method loadObject.

public CachedObjectWrapper<Document, Long> loadObject(Integer docNum, int start, int end, LayeredCache<Integer, Document, Long, ParsedDoc> cache) throws IOException {
    Document document = new Document();
    Get documentGet = new Get(SolbaseUtil.randomize(docNum));
    if (fieldNames == null || fieldNames.size() == 0) {
        // get all columns ( except this skips meta info )
        documentGet.addFamily(Bytes.toBytes("field"));
    } else {
        for (byte[] fieldName : fieldNames) {
            documentGet.addColumn(Bytes.toBytes("field"), fieldName);
        }
    }
    Result documentResult = null;
    // if docTable is set up, reuse instance, otherwise create brand new one and close after done
    if (this.docTable == null) {
        HTableInterface docTable = null;
        try {
            docTable = SolbaseUtil.getDocTable();
            documentResult = docTable.get(documentGet);
        } finally {
            SolbaseUtil.releaseTable(docTable);
        }
    } else {
        documentResult = this.docTable.get(documentGet);
    }
    if (documentResult == null || documentResult.isEmpty()) {
        return null;
    }
    // TODO, get from result
    Long versionIdentifier = 0l;
    NavigableMap<byte[], byte[]> familyMap = documentResult.getFamilyMap(Bytes.toBytes("field"));
    for (Map.Entry<byte[], byte[]> fieldColumn : familyMap.entrySet()) {
        Field field = null;
        String fieldName = Bytes.toString(fieldColumn.getKey());
        byte[] value;
        ByteBuffer v = ByteBuffer.wrap(fieldColumn.getValue());
        int vlimit = v.limit() + v.arrayOffset();
        if (v.array()[vlimit - 1] != Byte.MAX_VALUE && v.array()[vlimit - 1] != Byte.MIN_VALUE) {
            throw new CorruptIndexException("Solbase field is not properly encoded: " + docNum + "(" + fieldName + ")");
        } else if (v.array()[vlimit - 1] == Byte.MAX_VALUE) {
            // Binary
            value = new byte[vlimit - 1];
            System.arraycopy(v.array(), v.position() + v.arrayOffset(), value, 0, vlimit - 1);
            field = new Field(fieldName, value, Store.YES);
            document.add(field);
        } else if (v.array()[vlimit - 1] == Byte.MIN_VALUE) {
            // String
            value = new byte[vlimit - 1];
            System.arraycopy(v.array(), v.position() + v.arrayOffset(), value, 0, vlimit - 1);
            // Check for multi-fields
            String fieldString = new String(value, "UTF-8");
            if (fieldString.indexOf(Bytes.toString(SolbaseUtil.delimiter)) >= 0) {
                StringTokenizer tok = new StringTokenizer(fieldString, Bytes.toString(SolbaseUtil.delimiter));
                while (tok.hasMoreTokens()) {
                    // update logic
                    if (schema != null) {
                        SchemaField sfield = schema.getFieldOrNull(fieldName);
                        if (sfield.getType() instanceof EmbeddedIndexedIntField) {
                            EmbeddedIndexedIntField eiif = (EmbeddedIndexedIntField) sfield.getType();
                            EmbeddedSortField sf = new EmbeddedSortField(fieldName, tok.nextToken(), Field.Store.YES, Field.Index.NO, eiif.getFieldNumber());
                            document.add(sf);
                        } else {
                            Field f = sfield.createField(tok.nextToken(), 1.0f);
                            if (f != null) {
                                // null fields are not added
                                document.add(f);
                            }
                        }
                    } else {
                        field = new Field(fieldName, tok.nextToken(), Store.YES, Index.ANALYZED);
                        document.add(field);
                    }
                }
            } else {
                // update logic
                if (schema != null) {
                    SchemaField sfield = schema.getFieldOrNull(fieldName);
                    if (sfield.getType() instanceof EmbeddedIndexedIntField) {
                        EmbeddedIndexedIntField eiif = (EmbeddedIndexedIntField) sfield.getType();
                        EmbeddedSortField sf = new EmbeddedSortField(fieldName, fieldString, Field.Store.YES, Field.Index.NO, eiif.getFieldNumber());
                        document.add(sf);
                    } else {
                        Field f = sfield.createField(fieldString, 1.0f);
                        if (f != null) {
                            // null fields are not added
                            document.add(f);
                        }
                    }
                } else {
                    field = new Field(fieldName, fieldString, Store.YES, Index.ANALYZED);
                    document.add(field);
                }
            }
        }
    }
    return new CachedObjectWrapper<Document, Long>(document, versionIdentifier, System.currentTimeMillis());
}

Example 57 with NavigableMap

use of java.util.NavigableMap in project hbase by apache.

the class TestHBaseFsckReplicas method testNotInHdfsWithReplicas.

/**
   * This creates and fixes a bad table with a region that is in meta but has
   * no deployment or data hdfs. The table has region_replication set to 2.
   */
@Test(timeout = 180000)
public void testNotInHdfsWithReplicas() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    try {
        HRegionInfo[] oldHris = new HRegionInfo[2];
        setupTableWithRegionReplica(tableName, 2);
        assertEquals(ROWKEYS.length, countRows());
        NavigableMap<HRegionInfo, ServerName> map = MetaTableAccessor.allTableRegions(TEST_UTIL.getConnection(), tbl.getName());
        int i = 0;
        // store the HRIs of the regions we will mess up
        for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
            if (m.getKey().getStartKey().length > 0 && m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
                LOG.debug("Initially server hosting " + m.getKey() + " is " + m.getValue());
                oldHris[i++] = m.getKey();
            }
        }
        // make sure data in regions
        admin.flush(tableName);
        // Mess it up by leaving a hole in the hdfs data
        deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), false, false, // don't rm meta
        true);
        HBaseFsck hbck = doFsck(conf, false);
        assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_HDFS });
        // fix hole
        doFsck(conf, true);
        // check that hole fixed
        assertNoErrors(doFsck(conf, false));
        assertEquals(ROWKEYS.length - 2, countRows());
        // the following code checks whether the old primary/secondary has
        // been unassigned and the new primary/secondary has been assigned
        i = 0;
        HRegionInfo[] newHris = new HRegionInfo[2];
        // get all table's regions from meta
        map = MetaTableAccessor.allTableRegions(TEST_UTIL.getConnection(), tbl.getName());
        // get the HRIs of the new regions (hbck created new regions for fixing the hdfs mess-up)
        for (Map.Entry<HRegionInfo, ServerName> m : map.entrySet()) {
            if (m.getKey().getStartKey().length > 0 && m.getKey().getStartKey()[0] == Bytes.toBytes("B")[0]) {
                newHris[i++] = m.getKey();
            }
        }
        // get all the online regions in the regionservers
        Collection<ServerName> servers = admin.getClusterStatus().getServers();
        Set<HRegionInfo> onlineRegions = new HashSet<>();
        for (ServerName s : servers) {
            List<HRegionInfo> list = admin.getOnlineRegions(s);
            onlineRegions.addAll(list);
        }
        // the new HRIs must be a subset of the online regions
        assertTrue(onlineRegions.containsAll(Arrays.asList(newHris)));
        // the old HRIs must not be part of the set (removeAll would return false if
        // the set didn't change)
        assertFalse(onlineRegions.removeAll(Arrays.asList(oldHris)));
    } finally {
        cleanupTable(tableName);
        admin.close();
    }
}

Example 58 with NavigableMap

use of java.util.NavigableMap in project voltdb by VoltDB.

the class NodeSettings method archiveSnapshotDirectory.

default default boolean archiveSnapshotDirectory() {
    File snapshotDH = resolve(getSnapshoth());
    String[] snapshots = snapshotDH.list();
    if (snapshots == null || snapshots.length == 0) {
        return false;
    }
    Pattern archvRE = Pattern.compile(getSnapshoth().getName() + "\\.(\\d+)");
    final ImmutableSortedMap.Builder<Integer, File> mb = ImmutableSortedMap.naturalOrder();
    File parent = snapshotDH.getParentFile();
    parent.listFiles(new FileFilter() {

        @Override
        public boolean accept(File path) {
            Matcher mtc = archvRE.matcher(path.getName());
            if (path.isDirectory() && mtc.matches()) {
                mb.put(Integer.parseInt(mtc.group(1)), path);
                return true;
            }
            return false;
        }
    });
    NavigableMap<Integer, File> snapdirs = mb.build();
    for (Map.Entry<Integer, File> e : snapdirs.descendingMap().entrySet()) {
        File renameTo = new File(snapshotDH.getPath() + "." + (e.getKey() + 1));
        try {
            Files.move(e.getValue().toPath(), renameTo.toPath());
        } catch (IOException exc) {
            throw new SettingsException("failed to rename " + e.getValue() + " to " + renameTo, exc);
        }
    }
    File renameTo = new File(snapshotDH.getPath() + ".1");
    try {
        Files.move(snapshotDH.toPath(), renameTo.toPath());
    } catch (IOException e) {
        throw new SettingsException("failed to rename " + snapshotDH + " to " + renameTo, e);
    }
    if (!snapshotDH.mkdir()) {
        throw new SettingsException("failed to create snapshot directory " + snapshotDH);
    }
    return true;
}

Example 59 with NavigableMap

use of java.util.NavigableMap in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testResetCursor.

@Test(timeOut = 10000, dataProvider = "subType")
public void testResetCursor(SubscriptionType subType) throws Exception {
    final RetentionPolicies policy = new RetentionPolicies(60, 52 * 1024);
    final DestinationName destName = DestinationName.get("persistent://my-property/use/my-ns/unacked-topic");
    final int warmup = 20;
    final int testSize = 150;
    final List<Message> received = new ArrayList<Message>();
    final ConsumerConfiguration consConfig = new ConsumerConfiguration();
    final String subsId = "sub";
    final NavigableMap<Long, TimestampEntryCount> publishTimeIdMap = new ConcurrentSkipListMap<>();
    consConfig.setSubscriptionType(subType);
    consConfig.setMessageListener((MessageListener) (Consumer consumer, Message msg) -> {
        try {
            synchronized (received) {
                received.add(msg);
            }
            consumer.acknowledge(msg);
            long publishTime = ((MessageImpl) msg).getPublishTime();
            log.info(" publish time is " + publishTime + "," + msg.getMessageId());
            TimestampEntryCount timestampEntryCount = publishTimeIdMap.computeIfAbsent(publishTime, (k) -> new TimestampEntryCount(publishTime));
            timestampEntryCount.incrementAndGet();
        } catch (final PulsarClientException e) {
            log.warn("Failed to ack!");
        }
    });
    admin.namespaces().setRetention(destName.getNamespace(), policy);
    Consumer consumer = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
    final Producer producer = pulsarClient.createProducer(destName.toString());
    log.info("warm up started for " + destName.toString());
    // send warmup msgs
    byte[] msgBytes = new byte[1000];
    for (Integer i = 0; i < warmup; i++) {
        producer.send(msgBytes);
    }
    log.info("warm up finished.");
    // sleep to ensure receiving of msgs
    for (int n = 0; n < 10 && received.size() < warmup; n++) {
        Thread.sleep(100);
    }
    // validate received msgs
    Assert.assertEquals(received.size(), warmup);
    received.clear();
    // publish testSize num of msgs
    log.info("Sending more messages.");
    for (Integer n = 0; n < testSize; n++) {
        producer.send(msgBytes);
        Thread.sleep(1);
    }
    log.info("Sending more messages done.");
    Thread.sleep(3000);
    long begints = publishTimeIdMap.firstEntry().getKey();
    long endts = publishTimeIdMap.lastEntry().getKey();
    // find reset timestamp
    long timestamp = (endts - begints) / 2 + begints;
    timestamp = publishTimeIdMap.floorKey(timestamp);
    NavigableMap<Long, TimestampEntryCount> expectedMessages = new ConcurrentSkipListMap<>();
    expectedMessages.putAll(publishTimeIdMap.tailMap(timestamp, true));
    received.clear();
    log.info("reset cursor to " + timestamp + " for topic " + destName.toString() + " for subs " + subsId);
    log.info("issuing admin operation on " + admin.getServiceUrl().toString());
    List<String> subList = admin.persistentTopics().getSubscriptions(destName.toString());
    for (String subs : subList) {
        log.info("got sub " + subs);
    }
    publishTimeIdMap.clear();
    // reset the cursor to this timestamp
    Assert.assertTrue(subList.contains(subsId));
    admin.persistentTopics().resetCursor(destName.toString(), subsId, timestamp);
    consumer = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
    Thread.sleep(3000);
    int totalExpected = 0;
    for (TimestampEntryCount tec : expectedMessages.values()) {
        totalExpected += tec.numMessages;
    }
    // validate that replay happens after the timestamp
    Assert.assertTrue(publishTimeIdMap.firstEntry().getKey() >= timestamp);
    consumer.close();
    producer.close();
    // validate that expected and received counts match
    int totalReceived = 0;
    for (TimestampEntryCount tec : publishTimeIdMap.values()) {
        totalReceived += tec.numMessages;
    }
    Assert.assertEquals(totalReceived, totalExpected, "did not receive all messages on replay after reset");
}

Example 60 with NavigableMap

use of java.util.NavigableMap in project jdk8u_jdk by JetBrains.

the class EmptyNavigableMap method testSubMapRanges.

@Test(dataProvider = "NavigableMap<?,?>", dataProviderClass = EmptyNavigableMap.class)
public void testSubMapRanges(String description, NavigableMap navigableMap) {
    Object first = isDescending(navigableMap) ? BigInteger.TEN : BigInteger.ZERO;
    Object last = (BigInteger.ZERO == first) ? BigInteger.TEN : BigInteger.ZERO;
    NavigableMap subMap = navigableMap.subMap(first, true, last, true);
    // same subset
    subMap.subMap(first, true, last, true);
    // slightly smaller
    NavigableMap ns = subMap.subMap(first, false, last, false);
    // slight exapansion
    assertThrows(() -> {
        ns.subMap(first, true, last, true);
    }, IllegalArgumentException.class, description + ": Expansion should not be allowed");
    // much smaller
    subMap.subMap(first, false, BigInteger.ONE, false);
}