Examples with ReferenceFileRegion com.ctrip.xpipe.netty.filechannel.ReferenceFileRegion used on opensource projects

Example 1 with ReferenceFileRegion

use of com.ctrip.xpipe.netty.filechannel.ReferenceFileRegion in project x-pipe by ctripcorp.

the class DefaultCommandStoreTest method testGetAsSoonAsMessageWritten.

@Test
public void testGetAsSoonAsMessageWritten() throws IOException, InterruptedException {
    final StringBuilder sb = new StringBuilder();
    final Semaphore semaphore = new Semaphore(0);
    executors.execute(new AbstractExceptionLogTask() {

        @Override
        protected void doRun() throws Exception {
            commandStore.addCommandsListener(0, new CommandsListener() {

                @Override
                public ChannelFuture onCommand(ReferenceFileRegion referenceFileRegion) {
                    sb.append(readFileChannelInfoMessageAsString(referenceFileRegion));
                    semaphore.release();
                    return null;
                }

                @Override
                public boolean isOpen() {
                    return true;
                }

                @Override
                public void beforeCommand() {
                }
            });
        }
    });
    StringBuilder expected = new StringBuilder();
    for (int i = 0; i < (1 << 10); i++) {
        byte random = (byte) randomInt('a', 'z');
        semaphore.drainPermits();
        expected.append((char) random);
        commandStore.appendCommands(Unpooled.wrappedBuffer(new byte[] { random }));
        Assert.assertTrue(semaphore.tryAcquire(1000, TimeUnit.MILLISECONDS));
        logger.debug("{}", sb);
        Assert.assertEquals(expected.toString(), sb.toString());
    }
}

Example 2 with ReferenceFileRegion

use of com.ctrip.xpipe.netty.filechannel.ReferenceFileRegion in project x-pipe by ctripcorp.

the class DefaultCommandStoreTest method readCommandStoreTilNoMessage.

private String readCommandStoreTilNoMessage(final long offset, final DefaultCommandStore commandStore, int expectedLength) throws InterruptedException {
    final StringBuilder result = new StringBuilder();
    final Semaphore semaphore = new Semaphore(-expectedLength + 1);
    new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                commandStore.addCommandsListener(offset, new CommandsListener() {

                    @Override
                    public ChannelFuture onCommand(ReferenceFileRegion referenceFileRegion) {
                        logger.debug("[onCommand]{}", referenceFileRegion);
                        result.append(readFileChannelInfoMessageAsString(referenceFileRegion));
                        semaphore.release((int) referenceFileRegion.count());
                        return null;
                    }

                    @Override
                    public boolean isOpen() {
                        return true;
                    }

                    @Override
                    public void beforeCommand() {
                    }
                });
            } catch (IOException e) {
                logger.error("[run]", e);
            }
        }
    }).start();
    semaphore.tryAcquire(10, TimeUnit.SECONDS);
    return result.toString();
}

Example 3 with ReferenceFileRegion

use of com.ctrip.xpipe.netty.filechannel.ReferenceFileRegion in project x-pipe by ctripcorp.

the class AbstractRedisKeeperTest method readCommandFileTilEnd.

public String readCommandFileTilEnd(final long beginOffset, final ReplicationStore replicationStore, int expectedLen) throws IOException {
    final ByteArrayOutputStream baous = new ByteArrayOutputStream();
    new Thread() {

        public void run() {
            try {
                doRun();
            } catch (Exception e) {
                logger.error("[run]", e);
            }
        }

        private void doRun() throws IOException {
            replicationStore.addCommandsListener(replicationStore.beginOffsetWhenCreated() + beginOffset, new CommandsListener() {

                @Override
                public boolean isOpen() {
                    return true;
                }

                @Override
                public void beforeCommand() {
                }

                @Override
                public ChannelFuture onCommand(ReferenceFileRegion referenceFileRegion) {
                    try {
                        byte[] message = readFileChannelInfoMessageAsBytes(referenceFileRegion);
                        baous.write(message);
                    } catch (IOException e) {
                        logger.error("[onCommand]" + referenceFileRegion, e);
                    }
                    return null;
                }
            });
        }
    }.start();
    int lastSize = baous.size();
    long equalCount = 0;
    while (true) {
        int currentSize = baous.size();
        if (expectedLen >= 0 && currentSize >= expectedLen) {
            break;
        }
        if (currentSize != lastSize) {
            lastSize = currentSize;
            equalCount = 0;
        } else {
            equalCount++;
        }
        if (equalCount > 100) {
            break;
        }
        sleep(10);
    }
    return new String(baous.toByteArray());
}

Example 4 with ReferenceFileRegion

use of com.ctrip.xpipe.netty.filechannel.ReferenceFileRegion in project x-pipe by ctripcorp.

the class DefaultRdbStore method doReadRdbFile.

private void doReadRdbFile(RdbFileListener rdbFileListener, ReferenceFileChannel referenceFileChannel) throws IOException {
    rdbFileListener.setRdbFileInfo(eofType, rdbOffset);
    long lastLogTime = System.currentTimeMillis();
    while (rdbFileListener.isOpen() && (isRdbWriting(status.get()) || (status.get() == Status.Success && referenceFileChannel.hasAnythingToRead()))) {
        ReferenceFileRegion referenceFileRegion = referenceFileChannel.readTilEnd();
        rdbFileListener.onFileData(referenceFileRegion);
        if (referenceFileRegion.count() <= 0)
            try {
                Thread.sleep(1);
                long currentTime = System.currentTimeMillis();
                if (currentTime - lastLogTime > 10000) {
                    logger.info("[doReadRdbFile]status:{}, referenceFileChannel:{}, count:{}, rdbFileListener:{}", status.get(), referenceFileChannel, referenceFileRegion.count(), rdbFileListener);
                    lastLogTime = currentTime;
                }
            } catch (InterruptedException e) {
                logger.error("[doReadRdbFile]" + rdbFileListener, e);
                Thread.currentThread().interrupt();
            }
    }
    logger.info("[doReadRdbFile] done with status {}", status.get());
    switch(status.get()) {
        case Success:
            if (file.exists()) {
                // this is necessery because file may be deleted
                rdbFileListener.onFileData(null);
            } else {
                rdbFileListener.exception((new Exception("rdb file not exists now " + file)));
            }
            break;
        case Fail:
            rdbFileListener.exception(new Exception("[rdb error]" + file));
            break;
        default:
            rdbFileListener.exception(new Exception("[status not right]" + file + "," + status));
            break;
    }
}

Example 5 with ReferenceFileRegion

use of com.ctrip.xpipe.netty.filechannel.ReferenceFileRegion in project x-pipe by ctripcorp.

the class DefaultCommandStore method addCommandsListener.

@Override
public void addCommandsListener(long offset, final CommandsListener listener) throws IOException {
    makeSureOpen();
    logger.info("[addCommandsListener][begin] from offset {}, {}", offset, listener);
    CommandReader cmdReader = null;
    try {
        cmdReader = beginRead(offset);
    } finally {
        // ensure beforeCommand() is always called
        listener.beforeCommand();
    }
    logger.info("[addCommandsListener] from offset {}, {}", offset, cmdReader);
    try {
        while (listener.isOpen() && !Thread.currentThread().isInterrupted()) {
            final ReferenceFileRegion referenceFileRegion = cmdReader.read();
            logger.debug("[addCommandsListener] {}", referenceFileRegion);
            if (delayTraceLogger.isDebugEnabled()) {
                delayTraceLogger.debug("[write][begin]{}, {}", listener, referenceFileRegion.getTotalPos());
            }
            commandStoreDelay.beginSend(listener, referenceFileRegion.getTotalPos());
            ChannelFuture future = listener.onCommand(referenceFileRegion);
            if (future != null) {
                future.addListener(new ChannelFutureListener() {

                    @Override
                    public void operationComplete(ChannelFuture future) throws Exception {
                        commandStoreDelay.flushSucceed(listener, referenceFileRegion.getTotalPos());
                        if (logger.isDebugEnabled()) {
                            delayTraceLogger.debug("[write][ end ]{}, {}", listener, referenceFileRegion.getTotalPos());
                        }
                    }
                });
            }
            if (referenceFileRegion.count() <= 0) {
                try {
                    Thread.sleep(1);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
        }
    } catch (Throwable th) {
        logger.error("[readCommands][exit]" + listener, th);
    } finally {
        cmdReader.close();
    }
    logger.info("[addCommandsListener][end] from offset {}, {}", offset, listener);
}