Search in sources :

Example 41 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class MVCCStoreFactoryImplTest method setup.

@Before
public void setup() throws IOException {
    this.namespace = mock(Namespace.class);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    when(dlm.asyncClose()).thenReturn(FutureUtils.Void());
    when(namespace.openLog(anyString())).thenReturn(dlm);
    AsyncLogWriter logWriter = mock(AsyncLogWriter.class);
    when(dlm.openAsyncLogWriter()).thenReturn(FutureUtils.value(logWriter));
    when(logWriter.getLastTxId()).thenReturn(-1L);
    DLSN dlsn = new DLSN(0L, 0L, 0L);
    when(logWriter.write(any(LogRecord.class))).thenReturn(FutureUtils.value(dlsn));
    when(logWriter.asyncClose()).thenReturn(FutureUtils.Void());
    AsyncLogReader logReader = mock(AsyncLogReader.class);
    when(dlm.openAsyncLogReader(anyLong())).thenReturn(FutureUtils.value(logReader));
    when(logReader.asyncClose()).thenReturn(FutureUtils.Void());
    LogRecordWithDLSN record = new LogRecordWithDLSN(dlsn, 0L, NOP_CMD.toByteArray(), 0L);
    when(logReader.readNext()).thenReturn(FutureUtils.value(record));
    int numDirs = 3;
    this.storeDirs = new File[numDirs];
    for (int i = 0; i < numDirs; i++) {
        storeDirs[i] = testDir.newFolder("test-" + i);
    }
    this.resources = StorageResources.create(StorageResourcesSpec.builder().numCheckpointThreads(3).numIOReadThreads(3).numIOWriteThreads(3).build());
    this.factory = new MVCCStoreFactoryImpl(() -> namespace, storeDirs, resources, false);
}
Also used : LogRecordWithDLSN(org.apache.distributedlog.LogRecordWithDLSN) DLSN(org.apache.distributedlog.DLSN) LogRecordWithDLSN(org.apache.distributedlog.LogRecordWithDLSN) LogRecord(org.apache.distributedlog.LogRecord) AsyncLogReader(org.apache.distributedlog.api.AsyncLogReader) DistributedLogManager(org.apache.distributedlog.api.DistributedLogManager) AsyncLogWriter(org.apache.distributedlog.api.AsyncLogWriter) Namespace(org.apache.distributedlog.api.namespace.Namespace) Before(org.junit.Before)

Example 42 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class MVCCAsyncStoreTestBase method mockNamespace.

private static Namespace mockNamespace() throws Exception {
    Namespace namespace = mock(Namespace.class);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    when(dlm.asyncClose()).thenReturn(FutureUtils.Void());
    when(namespace.openLog(anyString())).thenReturn(dlm);
    AsyncLogWriter logWriter = mock(AsyncLogWriter.class);
    when(dlm.openAsyncLogWriter()).thenReturn(FutureUtils.value(logWriter));
    when(logWriter.getLastTxId()).thenReturn(-1L);
    DLSN dlsn = new DLSN(0L, 0L, 0L);
    when(logWriter.write(any(LogRecord.class))).thenReturn(FutureUtils.value(dlsn));
    when(logWriter.asyncClose()).thenReturn(FutureUtils.Void());
    AsyncLogReader logReader = mock(AsyncLogReader.class);
    when(dlm.openAsyncLogReader(anyLong())).thenReturn(FutureUtils.value(logReader));
    when(logReader.asyncClose()).thenReturn(FutureUtils.Void());
    LogRecordWithDLSN record = new LogRecordWithDLSN(dlsn, 0L, NOP_CMD.toByteArray(), 0L);
    when(logReader.readNext()).thenReturn(FutureUtils.value(record));
    return namespace;
}
Also used : LogRecordWithDLSN(org.apache.distributedlog.LogRecordWithDLSN) DLSN(org.apache.distributedlog.DLSN) LogRecordWithDLSN(org.apache.distributedlog.LogRecordWithDLSN) LogRecord(org.apache.distributedlog.LogRecord) AsyncLogReader(org.apache.distributedlog.api.AsyncLogReader) DistributedLogManager(org.apache.distributedlog.api.DistributedLogManager) AsyncLogWriter(org.apache.distributedlog.api.AsyncLogWriter) Namespace(org.apache.distributedlog.api.namespace.Namespace)

Example 43 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project incubator-heron by apache.

the class DLDownloader method download.

@Override
public void download(URI uri, Path destination) throws Exception {
    String path = uri.getPath();
    File pathFile = new File(path);
    String logName = pathFile.getName();
    String parentName = pathFile.getParent();
    URI parentUri = new URI(uri.getScheme(), uri.getAuthority(), parentName, uri.getQuery(), uri.getFragment());
    Namespace ns = builder.clientId("heron-downloader").conf(CONF).uri(parentUri).build();
    try {
        // open input stream
        InputStream in = openInputStream(ns, logName);
        Extractor.extract(in, destination);
    } finally {
        ns.close();
    }
}
Also used : DLInputStream(com.twitter.heron.dlog.DLInputStream) InputStream(java.io.InputStream) File(java.io.File) URI(java.net.URI) Namespace(org.apache.distributedlog.api.namespace.Namespace)

Example 44 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project incubator-heron by apache.

the class DlogUploaderTest method testUploadPackageExisting.

@Test
public void testUploadPackageExisting() throws Exception {
    uploader = Mockito.spy(uploader);
    Namespace ns = mock(Namespace.class);
    when(nsBuilder.build()).thenReturn(ns);
    when(ns.logExists(anyString())).thenReturn(true);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    when(ns.openLog(anyString())).thenReturn(dlm);
    AppendOnlyStreamWriter asw = mock(AppendOnlyStreamWriter.class);
    when(dlm.getAppendOnlyStreamWriter()).thenReturn(asw);
    Mockito.doReturn(true).when(uploader).isLocalFileExists(Mockito.anyString());
    uploader.initialize(config);
    uploader.uploadPackage();
    verify(ns, times(1)).deleteLog(eq(uploader.getPackageName()));
    verify(copier, times(1)).copyFileToStream(eq(uploader.getTopologyPackageLocation()), any(OutputStream.class));
    verify(asw, times(1)).close();
    verify(dlm, times(1)).close();
}
Also used : DistributedLogManager(org.apache.distributedlog.api.DistributedLogManager) AppendOnlyStreamWriter(org.apache.distributedlog.AppendOnlyStreamWriter) OutputStream(java.io.OutputStream) Namespace(org.apache.distributedlog.api.namespace.Namespace) Test(org.junit.Test)

Example 45 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project incubator-heron by apache.

the class DlogUploaderTest method testClose.

@Test
public void testClose() throws Exception {
    Namespace ns = mock(Namespace.class);
    when(nsBuilder.build()).thenReturn(ns);
    uploader.initialize(config);
    uploader.close();
    verify(ns, times(1)).close();
}
Also used : Namespace(org.apache.distributedlog.api.namespace.Namespace) Test(org.junit.Test)

Aggregations

Namespace (org.apache.distributedlog.api.namespace.Namespace)50 Test (org.junit.Test)31 URI (java.net.URI)26 DistributedLogManager (org.apache.distributedlog.api.DistributedLogManager)25 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)13 InputStream (java.io.InputStream)12 IOException (java.io.IOException)10 Response (javax.ws.rs.core.Response)8 Test (org.testng.annotations.Test)8 FunctionMetaData (org.apache.pulsar.functions.proto.Function.FunctionMetaData)6 RequestResult (org.apache.pulsar.functions.worker.request.RequestResult)6 AsyncLogReader (org.apache.distributedlog.api.AsyncLogReader)5 AsyncLogWriter (org.apache.distributedlog.api.AsyncLogWriter)5 LogWriter (org.apache.distributedlog.api.LogWriter)5 ErrorData (org.apache.pulsar.common.policies.data.ErrorData)5 DLSN (org.apache.distributedlog.DLSN)4 DistributedLogConfiguration (org.apache.distributedlog.DistributedLogConfiguration)4 Matchers.anyString (org.mockito.Matchers.anyString)4 DLInputStream (com.twitter.heron.dlog.DLInputStream)3 OutputStream (java.io.OutputStream)3