Examples with DLSN com.twitter.distributedlog.DLSN used on opensource projects

Example 11 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
     * {@link https://issues.apache.org/jira/browse/DL-44}
     */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = factory.createDistributedLogManagerWithSharedClients(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecord record = Await.result(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    DLSN dlsn = readDLM.getLastDLSN();
    assertTrue(dlsn.compareTo(new DLSN(5, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(4, -1, Long.MIN_VALUE)) > 0);
    // there isn't records should be read
    Future<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        Await.result(readFuture, Duration.fromMilliseconds(1000));
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (TimeoutException te) {
    // expected
    }
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), streamName, false, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    dlsn = readDLM.getLastDLSN();
    assertTrue(dlsn.compareTo(new DLSN(5, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(4, -1, Long.MIN_VALUE)) > 0);
    // there isn't records should be read
    try {
        Await.result(readFuture, Duration.fromMilliseconds(1000));
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (TimeoutException te) {
    // expected
    }
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), streamName, false, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    expectedTxId = 51L;
    LogRecord record = Await.result(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Await.result(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    dlsn = readDLM.getLastDLSN();
    LOG.info("LastDLSN after fix inprogress segment : {}", dlsn);
    assertTrue(dlsn.compareTo(new DLSN(7, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(6, -1, Long.MIN_VALUE)) > 0);
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    factory.close();
}

Example 12 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testFailRequestAfterRetriedAllStreams.

@Test(timeout = 20000)
public void testFailRequestAfterRetriedAllStreams() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())).thenReturn(new Promise<DLSN>());
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(10).maxSpeculativeTimeoutMs(20).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(10).bufferSize(Integer.MAX_VALUE).build();
    byte[] data = "test-test".getBytes(UTF_8);
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    try {
        Await.result(writeFuture);
        fail("Should fail the request after retries all streams");
    } catch (IndividualRequestTimeoutException e) {
        long timeoutMs = e.timeout().inMilliseconds();
        assertTrue(timeoutMs >= (10 + 20) && timeoutMs < 5000000);
    }
    writer.close();
}

Example 13 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testPeriodicalFlush.

@Test(timeout = 20000)
public void testPeriodicalFlush() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(10).maxSpeculativeTimeoutMs(20).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(10).bufferSize(Integer.MAX_VALUE).build();
    final DLSN dlsn = new DLSN(99L, 88L, 0L);
    Mockito.doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            return Future.value(dlsn);
        }
    }).when(client).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    byte[] data = "test-test".getBytes(UTF_8);
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    DLSN writeDLSN = Await.result(writeFuture);
    assertEquals(dlsn, writeDLSN);
    writer.close();
}

Example 14 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testFlushWhenExceedMaxLogRecordSetSize.

@Test(timeout = 20000)
public void testFlushWhenExceedMaxLogRecordSetSize() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())).thenReturn(Future.value(new DLSN(1L, 1L, 999L)));
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(100000).maxSpeculativeTimeoutMs(200000).speculativeBackoffMultiplier(2).requestTimeoutMs(500000).flushIntervalMs(0).bufferSize(Integer.MAX_VALUE).scheduler(executorService).build();
    byte[] data = new byte[LogRecord.MAX_LOGRECORD_SIZE - 3 * 100];
    ByteBuffer buffer1 = ByteBuffer.wrap(data);
    writer.write(buffer1);
    verify(client, times(0)).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    LogRecordSet.Writer recordSetWriter1 = writer.getLogRecordSetWriter();
    assertEquals(1, recordSetWriter1.getNumRecords());
    assertEquals(LogRecordSet.HEADER_LEN + 4 + data.length, recordSetWriter1.getNumBytes());
    ByteBuffer buffer2 = ByteBuffer.wrap(data);
    writer.write(buffer2);
    verify(client, times(1)).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    LogRecordSet.Writer recordSetWriter2 = writer.getLogRecordSetWriter();
    assertEquals(1, recordSetWriter2.getNumRecords());
    assertEquals(LogRecordSet.HEADER_LEN + 4 + data.length, recordSetWriter2.getNumBytes());
    assertTrue(recordSetWriter1 != recordSetWriter2);
    writer.close();
}

Example 15 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class ReaderWorker method reinitStream.

private void reinitStream(int idx, Promise<Void> promise) {
    int streamId = startStreamId + idx;
    String streamName = String.format("%s_%d", streamPrefix, streamId);
    if (logReaders[idx] != null) {
        try {
            FutureUtils.result(logReaders[idx].asyncClose());
        } catch (IOException e) {
            LOG.warn("Failed on closing stream reader {} : ", streamName, e);
        }
        logReaders[idx] = null;
    }
    if (dlms[idx] != null) {
        try {
            dlms[idx].close();
        } catch (IOException e) {
            LOG.warn("Failed on closing dlm {} : ", streamName, e);
        }
        dlms[idx] = null;
    }
    try {
        dlms[idx] = namespace.openLog(streamName);
    } catch (IOException ioe) {
        LOG.error("Failed on creating dlm {} : ", streamName, ioe);
        scheduleReinitStream(idx, promise);
        return;
    }
    DLSN lastDLSN;
    if (readFromHead) {
        lastDLSN = DLSN.InitialDLSN;
    } else {
        try {
            lastDLSN = dlms[idx].getLastDLSN();
        } catch (IOException ioe) {
            LOG.error("Failed on getting last dlsn from stream {} : ", streamName, ioe);
            scheduleReinitStream(idx, promise);
            return;
        }
    }
    try {
        logReaders[idx] = dlms[idx].getAsyncLogReader(lastDLSN);
    } catch (IOException ioe) {
        LOG.error("Failed on opening reader for stream {} starting from {} : ", new Object[] { streamName, lastDLSN, ioe });
        scheduleReinitStream(idx, promise);
        return;
    }
    LOG.info("Opened reader for stream {}, starting from {}.", streamName, lastDLSN);
    promise.setValue(null);
}