Examples with BookieSocketAddress org.apache.bookkeeper.net.BookieSocketAddress used on opensource projects

Example 96 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class UpdateLedgerOpTest method testManyLedgers.

public void testManyLedgers(boolean useShortHostName) throws Exception {
    BookKeeper bk = new BookKeeper(baseClientConf, zkc);
    BookKeeperAdmin bkadmin = new BookKeeperAdmin(bk);
    LOG.info("Create ledger and add entries to it");
    List<LedgerHandle> ledgers = new ArrayList<LedgerHandle>();
    LedgerHandle lh1 = createLedgerWithEntries(bk, 0);
    ledgers.add(lh1);
    for (int i = 0; i < 99; i++) {
        ledgers.add(createLedgerWithEntries(bk, 0));
    }
    ArrayList<BookieSocketAddress> ensemble = lh1.getLedgerMetadata().getEnsemble(0);
    BookieSocketAddress curBookieAddr = ensemble.get(0);
    baseConf.setUseHostNameAsBookieID(true);
    baseConf.setUseShortHostName(useShortHostName);
    BookieSocketAddress curBookieId = Bookie.getBookieAddress(baseConf);
    BookieSocketAddress toBookieAddr = new BookieSocketAddress(curBookieId.getHostName() + ":" + curBookieAddr.getPort());
    UpdateLedgerOp updateLedgerOp = new UpdateLedgerOp(bk, bkadmin);
    updateLedgerOp.updateBookieIdInLedgers(curBookieAddr, toBookieAddr, 5, Integer.MIN_VALUE, progressable);
    for (LedgerHandle lh : ledgers) {
        // ledger#close() would hit BadVersion exception as rename
        // increments cversion. But LedgerMetadata#isConflictWith()
        // gracefully handles this conflicts.
        lh.close();
        LedgerHandle openLedger = bk.openLedger(lh.getId(), digestType, PASSWORD.getBytes());
        ensemble = openLedger.getLedgerMetadata().getEnsemble(0);
        assertTrue("Failed to update the ledger metadata to use bookie host name", ensemble.contains(toBookieAddr));
        assertFalse("Failed to update the ledger metadata to use bookie host name", ensemble.contains(curBookieAddr));
    }
}

Example 97 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class BenchBookie method main.

/**
 * @param args
 * @throws InterruptedException
 */
public static void main(String[] args) throws InterruptedException, ParseException, IOException, BKException, KeeperException {
    Options options = new Options();
    options.addOption("host", true, "Hostname or IP of bookie to benchmark");
    options.addOption("port", true, "Port of bookie to benchmark (default 3181)");
    options.addOption("zookeeper", true, "Zookeeper ensemble, (default \"localhost:2181\")");
    options.addOption("size", true, "Size of message to send, in bytes (default 1024)");
    options.addOption("warmupCount", true, "Number of messages in warmup phase (default 999)");
    options.addOption("latencyCount", true, "Number of messages in latency phase (default 5000)");
    options.addOption("throughputCount", true, "Number of messages in throughput phase (default 50000)");
    options.addOption("help", false, "This message");
    CommandLineParser parser = new PosixParser();
    CommandLine cmd = parser.parse(options, args);
    if (cmd.hasOption("help") || !cmd.hasOption("host")) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("BenchBookie <options>", options);
        System.exit(-1);
    }
    String addr = cmd.getOptionValue("host");
    int port = Integer.parseInt(cmd.getOptionValue("port", "3181"));
    int size = Integer.parseInt(cmd.getOptionValue("size", "1024"));
    String servers = cmd.getOptionValue("zookeeper", "localhost:2181");
    int warmUpCount = Integer.parseInt(cmd.getOptionValue("warmupCount", "999"));
    int latencyCount = Integer.parseInt(cmd.getOptionValue("latencyCount", "5000"));
    int throughputCount = Integer.parseInt(cmd.getOptionValue("throughputCount", "50000"));
    EventLoopGroup eventLoop;
    if (SystemUtils.IS_OS_LINUX) {
        try {
            eventLoop = new EpollEventLoopGroup();
        } catch (Throwable t) {
            LOG.warn("Could not use Netty Epoll event loop for benchmark {}", t.getMessage());
            eventLoop = new NioEventLoopGroup();
        }
    } else {
        eventLoop = new NioEventLoopGroup();
    }
    OrderedExecutor executor = OrderedExecutor.newBuilder().name("BenchBookieClientScheduler").numThreads(1).build();
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("BookKeeperClientScheduler"));
    ClientConfiguration conf = new ClientConfiguration();
    BookieClient bc = new BookieClient(conf, eventLoop, executor, scheduler, NullStatsLogger.INSTANCE);
    LatencyCallback lc = new LatencyCallback();
    ThroughputCallback tc = new ThroughputCallback();
    long ledger = getValidLedgerId(servers);
    for (long entry = 0; entry < warmUpCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), tc, null, BookieProtocol.FLAG_NONE);
    }
    LOG.info("Waiting for warmup");
    tc.waitFor(warmUpCount);
    ledger = getValidLedgerId(servers);
    LOG.info("Benchmarking latency");
    long startTime = System.nanoTime();
    for (long entry = 0; entry < latencyCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        lc.resetComplete();
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), lc, null, BookieProtocol.FLAG_NONE);
        lc.waitForComplete();
    }
    long endTime = System.nanoTime();
    LOG.info("Latency: " + (((double) (endTime - startTime)) / ((double) latencyCount)) / 1000000.0);
    ledger = getValidLedgerId(servers);
    LOG.info("Benchmarking throughput");
    startTime = System.currentTimeMillis();
    tc = new ThroughputCallback();
    for (long entry = 0; entry < throughputCount; entry++) {
        ByteBuf toSend = Unpooled.buffer(size);
        toSend.resetReaderIndex();
        toSend.resetWriterIndex();
        toSend.writeLong(ledger);
        toSend.writeLong(entry);
        toSend.writerIndex(toSend.capacity());
        bc.addEntry(new BookieSocketAddress(addr, port), ledger, new byte[20], entry, ByteBufList.get(toSend), tc, null, BookieProtocol.FLAG_NONE);
    }
    tc.waitFor(throughputCount);
    endTime = System.currentTimeMillis();
    LOG.info("Throughput: " + ((long) throughputCount) * 1000 / (endTime - startTime));
    bc.close();
    scheduler.shutdown();
    eventLoop.shutdownGracefully();
    executor.shutdown();
}

Example 98 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class LedgerChecker method checkLedger.

public void checkLedger(final LedgerHandle lh, final GenericCallback<Set<LedgerFragment>> cb, long percentageOfLedgerFragmentToBeVerified) {
    // build a set of all fragment replicas
    final Set<LedgerFragment> fragments = new HashSet<LedgerFragment>();
    Long curEntryId = null;
    ArrayList<BookieSocketAddress> curEnsemble = null;
    for (Map.Entry<Long, ArrayList<BookieSocketAddress>> e : lh.getLedgerMetadata().getEnsembles().entrySet()) {
        if (curEntryId != null) {
            Set<Integer> bookieIndexes = new HashSet<Integer>();
            for (int i = 0; i < curEnsemble.size(); i++) {
                bookieIndexes.add(i);
            }
            fragments.add(new LedgerFragment(lh, curEntryId, e.getKey() - 1, bookieIndexes));
        }
        curEntryId = e.getKey();
        curEnsemble = e.getValue();
    }
    /* Checking the last segment of the ledger can be complicated in some cases.
         * In the case that the ledger is closed, we can just check the fragments of
         * the segment as normal even if no data has ever been written to.
         * In the case that the ledger is open, but enough entries have been written,
         * for lastAddConfirmed to be set above the start entry of the segment, we
         * can also check as normal.
         * However, if ledger is open, sometimes lastAddConfirmed cannot be trusted,
         * such as when it's lower than the first entry id, or not set at all,
         * we cannot be sure if there has been data written to the segment.
         * For this reason, we have to send a read request
         * to the bookies which should have the first entry. If they respond with
         * NoSuchEntry we can assume it was never written. If they respond with anything
         * else, we must assume the entry has been written, so we run the check.
         */
    if (curEntryId != null) {
        long lastEntry = lh.getLastAddConfirmed();
        if (!lh.isClosed() && lastEntry < curEntryId) {
            lastEntry = curEntryId;
        }
        Set<Integer> bookieIndexes = new HashSet<Integer>();
        for (int i = 0; i < curEnsemble.size(); i++) {
            bookieIndexes.add(i);
        }
        final LedgerFragment lastLedgerFragment = new LedgerFragment(lh, curEntryId, lastEntry, bookieIndexes);
        // Check for the case that no last confirmed entry has been set
        if (curEntryId == lastEntry) {
            final long entryToRead = curEntryId;
            final EntryExistsCallback eecb = new EntryExistsCallback(lh.getLedgerMetadata().getWriteQuorumSize(), new GenericCallback<Boolean>() {

                public void operationComplete(int rc, Boolean result) {
                    if (result) {
                        fragments.add(lastLedgerFragment);
                    }
                    checkFragments(fragments, cb, percentageOfLedgerFragmentToBeVerified);
                }
            });
            DistributionSchedule.WriteSet writeSet = lh.getDistributionSchedule().getWriteSet(entryToRead);
            for (int i = 0; i < writeSet.size(); i++) {
                BookieSocketAddress addr = curEnsemble.get(writeSet.get(i));
                bookieClient.readEntry(addr, lh.getId(), entryToRead, eecb, null, BookieProtocol.FLAG_NONE);
            }
            writeSet.recycle();
            return;
        } else {
            fragments.add(lastLedgerFragment);
        }
    }
    checkFragments(fragments, cb, percentageOfLedgerFragmentToBeVerified);
}

Example 99 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class LedgerHandle method replaceBookieInMetadata.

EnsembleInfo replaceBookieInMetadata(final Map<Integer, BookieSocketAddress> failedBookies, int ensembleChangeIdx) throws BKException.BKNotEnoughBookiesException {
    final ArrayList<BookieSocketAddress> newEnsemble = new ArrayList<BookieSocketAddress>();
    final long newEnsembleStartEntry = getLastAddConfirmed() + 1;
    final HashSet<Integer> replacedBookies = new HashSet<Integer>();
    synchronized (metadata) {
        newEnsemble.addAll(metadata.currentEnsemble);
        for (Map.Entry<Integer, BookieSocketAddress> entry : failedBookies.entrySet()) {
            int idx = entry.getKey();
            BookieSocketAddress addr = entry.getValue();
            if (LOG.isDebugEnabled()) {
                LOG.debug("[EnsembleChange-L{}-{}] : replacing bookie: {} index: {}", getId(), ensembleChangeIdx, addr, idx);
            }
            if (!newEnsemble.get(idx).equals(addr)) {
                // ensemble has already changed, failure of this addr is immaterial
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Write did not succeed to {}, bookieIndex {}, but we have already fixed it.", addr, idx);
                }
                continue;
            }
            try {
                BookieSocketAddress newBookie = bk.getBookieWatcher().replaceBookie(metadata.getEnsembleSize(), metadata.getWriteQuorumSize(), metadata.getAckQuorumSize(), metadata.getCustomMetadata(), newEnsemble, idx, new HashSet<BookieSocketAddress>(failedBookies.values()));
                newEnsemble.set(idx, newBookie);
                replacedBookies.add(idx);
            } catch (BKException.BKNotEnoughBookiesException e) {
                // if there is no bookie replaced, we throw not enough bookie exception
                if (replacedBookies.size() <= 0) {
                    throw e;
                } else {
                    break;
                }
            }
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("[EnsembleChange-L{}-{}] : changing ensemble from: {} to: {} starting at entry: {}," + " failed bookies: {}, replaced bookies: {}", ledgerId, ensembleChangeIdx, metadata.currentEnsemble, newEnsemble, (getLastAddConfirmed() + 1), failedBookies, replacedBookies);
        }
        metadata.addEnsemble(newEnsembleStartEntry, newEnsemble);
    }
    return new EnsembleInfo(newEnsemble, failedBookies, replacedBookies);
}

Example 100 with BookieSocketAddress

use of org.apache.bookkeeper.net.BookieSocketAddress in project bookkeeper by apache.

the class LedgerMetadata method parseConfig.

/**
 * Parses a given byte array and transforms into a LedgerConfig object.
 *
 * @param bytes
 *            byte array to parse
 * @param version
 *            version of the ledger metadata
 * @param msCtime
 *            metadata store creation time, used for legacy ledgers
 * @return LedgerConfig
 * @throws IOException
 *             if the given byte[] cannot be parsed
 */
public static LedgerMetadata parseConfig(byte[] bytes, Version version, Optional<Long> msCtime) throws IOException {
    LedgerMetadata lc = new LedgerMetadata();
    lc.version = version;
    String config = new String(bytes, UTF_8);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parsing Config: {}", config);
    }
    BufferedReader reader = new BufferedReader(new StringReader(config));
    String versionLine = reader.readLine();
    if (versionLine == null) {
        throw new IOException("Invalid metadata. Content missing");
    }
    if (versionLine.startsWith(VERSION_KEY)) {
        String[] parts = versionLine.split(tSplitter);
        lc.metadataFormatVersion = Integer.parseInt(parts[1]);
    } else {
        // if no version is set, take it to be version 1
        // as the parsing is the same as what we had before
        // we introduce versions
        lc.metadataFormatVersion = 1;
        // reset the reader
        reader.close();
        reader = new BufferedReader(new StringReader(config));
    }
    if (lc.metadataFormatVersion < LOWEST_COMPAT_METADATA_FORMAT_VERSION || lc.metadataFormatVersion > CURRENT_METADATA_FORMAT_VERSION) {
        throw new IOException("Metadata version not compatible. Expected between " + LOWEST_COMPAT_METADATA_FORMAT_VERSION + " and " + CURRENT_METADATA_FORMAT_VERSION + ", but got " + lc.metadataFormatVersion);
    }
    if (lc.metadataFormatVersion == 1) {
        return parseVersion1Config(lc, reader);
    }
    // remaining size is total minus the length of the version line and '\n'
    char[] configBuffer = new char[config.length() - (versionLine.length() + 1)];
    if (configBuffer.length != reader.read(configBuffer, 0, configBuffer.length)) {
        throw new IOException("Invalid metadata buffer");
    }
    LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder();
    TextFormat.merge((CharSequence) CharBuffer.wrap(configBuffer), builder);
    LedgerMetadataFormat data = builder.build();
    lc.writeQuorumSize = data.getQuorumSize();
    if (data.hasCtime()) {
        lc.ctime = data.getCtime();
        lc.storeSystemtimeAsLedgerCreationTime = true;
    } else if (msCtime.isPresent()) {
        lc.ctime = msCtime.get();
        lc.storeSystemtimeAsLedgerCreationTime = false;
    }
    if (data.hasAckQuorumSize()) {
        lc.ackQuorumSize = data.getAckQuorumSize();
    } else {
        lc.ackQuorumSize = lc.writeQuorumSize;
    }
    lc.ensembleSize = data.getEnsembleSize();
    lc.length = data.getLength();
    lc.state = data.getState();
    lc.lastEntryId = data.getLastEntryId();
    if (data.hasPassword()) {
        lc.digestType = data.getDigestType();
        lc.password = data.getPassword().toByteArray();
        lc.hasPassword = true;
    }
    for (LedgerMetadataFormat.Segment s : data.getSegmentList()) {
        ArrayList<BookieSocketAddress> addrs = new ArrayList<BookieSocketAddress>();
        for (String member : s.getEnsembleMemberList()) {
            addrs.add(new BookieSocketAddress(member));
        }
        lc.addEnsemble(s.getFirstEntryId(), addrs);
    }
    if (data.getCustomMetadataCount() > 0) {
        List<LedgerMetadataFormat.cMetadataMapEntry> cMetadataList = data.getCustomMetadataList();
        lc.customMetadata = Maps.newHashMap();
        for (LedgerMetadataFormat.cMetadataMapEntry ent : cMetadataList) {
            lc.customMetadata.put(ent.getKey(), ent.getValue().toByteArray());
        }
    }
    return lc;
}