Examples with PackagesStorage org.apache.pulsar.packages.management.core.PackagesStorage used on opensource projects

Example 1 with PackagesStorage

use of org.apache.pulsar.packages.management.core.PackagesStorage in project pulsar by apache.

the class PulsarService method startPackagesManagementService.

private void startPackagesManagementService() throws IOException {
    // TODO: using provider to initialize the packages management service.
    PackagesManagement packagesManagementService = new PackagesManagementImpl();
    this.packagesManagement = Optional.of(packagesManagementService);
    PackagesStorageProvider storageProvider = PackagesStorageProvider.newProvider(config.getPackagesManagementStorageProvider());
    DefaultPackagesStorageConfiguration storageConfiguration = new DefaultPackagesStorageConfiguration();
    storageConfiguration.setProperty(config.getProperties());
    PackagesStorage storage = storageProvider.getStorage(storageConfiguration);
    storage.initialize();
    packagesManagementService.initialize(storage);
}

Example 2 with PackagesStorage

use of org.apache.pulsar.packages.management.core.PackagesStorage in project pulsar by apache.

the class BookKeeperPackagesStorageTest method testReadWriteOperationsWithSeparatedBkCluster.

@Test(timeOut = 60000)
public void testReadWriteOperationsWithSeparatedBkCluster() throws Exception {
    PackagesStorageProvider provider = PackagesStorageProvider.newProvider(BookKeeperPackagesStorageProvider.class.getName());
    DefaultPackagesStorageConfiguration configuration = new DefaultPackagesStorageConfiguration();
    // set the unavailable bk cluster with mock zookeeper path
    configuration.setProperty("metadataStoreUrl", zkUtil.getZooKeeperConnectString() + "/mock");
    configuration.setProperty("packagesReplicas", "1");
    configuration.setProperty("packagesManagementLedgerRootPath", "/ledgers");
    PackagesStorage storage1 = provider.getStorage(configuration);
    storage1.initialize();
    String mockData = "mock-data";
    ByteArrayInputStream mockDataStream = new ByteArrayInputStream(mockData.getBytes(StandardCharsets.UTF_8));
    String mockPath = "mock-path";
    // write some data to the dlog will fail
    try {
        storage1.writeAsync(mockPath, mockDataStream).get();
    } catch (Exception e) {
        String errMsg = e.getCause().getMessage();
        assertTrue(errMsg.contains("Error on allocating ledger") || errMsg.contains("Write rejected"));
    } finally {
        storage1.closeAsync().get();
    }
    // set the available bk cluster with bookkeeperMetadataServiceUri using actual zookeeper path
    String bookkeeperMetadataServiceUri = String.format("zk+null://%s/ledgers", zkUtil.getZooKeeperConnectString());
    DefaultPackagesStorageConfiguration configuration2 = new DefaultPackagesStorageConfiguration();
    configuration2.setProperty("metadataStoreUrl", zkUtil.getZooKeeperConnectString());
    configuration2.setProperty("bookkeeperMetadataServiceUri", bookkeeperMetadataServiceUri);
    configuration2.setProperty("packagesReplicas", "1");
    PackagesStorage storage2 = provider.getStorage(configuration2);
    storage2.initialize();
    String testData = "test-data";
    ByteArrayInputStream testDataStream = new ByteArrayInputStream(testData.getBytes(StandardCharsets.UTF_8));
    String testPath = "test-path";
    // write some data to the dlog will success
    try {
        storage2.writeAsync(testPath, testDataStream).get();
        // read the data from the dlog
        ByteArrayOutputStream readData = new ByteArrayOutputStream();
        storage2.readAsync(testPath, readData).get();
        String readResult = new String(readData.toByteArray(), StandardCharsets.UTF_8);
        assertEquals(testData, readResult);
    } finally {
        storage2.closeAsync().get();
    }
}

Example 3 with PackagesStorage

use of org.apache.pulsar.packages.management.core.PackagesStorage in project pulsar by yahoo.

the class BookKeeperPackagesStorageTest method testReadWriteOperationsWithSeparatedBkCluster.

@Test(timeOut = 60000)
public void testReadWriteOperationsWithSeparatedBkCluster() throws Exception {
    PackagesStorageProvider provider = PackagesStorageProvider.newProvider(BookKeeperPackagesStorageProvider.class.getName());
    DefaultPackagesStorageConfiguration configuration = new DefaultPackagesStorageConfiguration();
    // set the unavailable bk cluster with mock zookeeper path
    configuration.setProperty("metadataStoreUrl", zkUtil.getZooKeeperConnectString() + "/mock");
    configuration.setProperty("packagesReplicas", "1");
    configuration.setProperty("packagesManagementLedgerRootPath", "/ledgers");
    PackagesStorage storage1 = provider.getStorage(configuration);
    storage1.initialize();
    String mockData = "mock-data";
    ByteArrayInputStream mockDataStream = new ByteArrayInputStream(mockData.getBytes(StandardCharsets.UTF_8));
    String mockPath = "mock-path";
    // write some data to the dlog will fail
    try {
        storage1.writeAsync(mockPath, mockDataStream).get();
    } catch (Exception e) {
        String errMsg = e.getCause().getMessage();
        assertTrue(errMsg.contains("Error on allocating ledger") || errMsg.contains("Write rejected"));
    } finally {
        storage1.closeAsync().get();
    }
    // set the available bk cluster with bookkeeperMetadataServiceUri using actual zookeeper path
    String bookkeeperMetadataServiceUri = String.format("zk+null://%s/ledgers", zkUtil.getZooKeeperConnectString());
    DefaultPackagesStorageConfiguration configuration2 = new DefaultPackagesStorageConfiguration();
    configuration2.setProperty("metadataStoreUrl", zkUtil.getZooKeeperConnectString());
    configuration2.setProperty("bookkeeperMetadataServiceUri", bookkeeperMetadataServiceUri);
    configuration2.setProperty("packagesReplicas", "1");
    PackagesStorage storage2 = provider.getStorage(configuration2);
    storage2.initialize();
    String testData = "test-data";
    ByteArrayInputStream testDataStream = new ByteArrayInputStream(testData.getBytes(StandardCharsets.UTF_8));
    String testPath = "test-path";
    // write some data to the dlog will success
    try {
        storage2.writeAsync(testPath, testDataStream).get();
        // read the data from the dlog
        ByteArrayOutputStream readData = new ByteArrayOutputStream();
        storage2.readAsync(testPath, readData).get();
        String readResult = new String(readData.toByteArray(), StandardCharsets.UTF_8);
        assertEquals(testData, readResult);
    } finally {
        storage2.closeAsync().get();
    }
}

Example 4 with PackagesStorage

use of org.apache.pulsar.packages.management.core.PackagesStorage in project pulsar by yahoo.

the class PackagesManagementImplTest method testPackagePath.

@Test
public void testPackagePath() {
    PackagesManagementImpl impl = (PackagesManagementImpl) packagesManagement;
    PackageName pn = PackageName.get("function://public/default/test@v1");
    String metaPath = impl.metadataPath(pn);
    Assert.assertEquals(metaPath, "function/public/default/test/v1/meta");
    String dataPath = impl.packagePath(pn);
    Assert.assertEquals(dataPath, "function/public/default/test/v1");
    impl.initialize(new PackagesStorage() {

        @Override
        public void initialize() {
        }

        @Override
        public CompletableFuture<Void> writeAsync(String path, InputStream inputStream) {
            return null;
        }

        @Override
        public CompletableFuture<Void> readAsync(String path, OutputStream outputStream) {
            return null;
        }

        @Override
        public CompletableFuture<Void> deleteAsync(String path) {
            return null;
        }

        @Override
        public CompletableFuture<List<String>> listAsync(String path) {
            return null;
        }

        @Override
        public CompletableFuture<Boolean> existAsync(String path) {
            return null;
        }

        @Override
        public CompletableFuture<Void> closeAsync() {
            return null;
        }

        @Override
        public String dataPath() {
            return "/tmp";
        }
    });
    metaPath = impl.metadataPath(pn);
    Assert.assertEquals(metaPath, "function/public/default/test/v1/meta");
    dataPath = impl.packagePath(pn);
    Assert.assertEquals(dataPath, "function/public/default/test/v1/tmp");
}

Example 5 with PackagesStorage

use of org.apache.pulsar.packages.management.core.PackagesStorage in project incubator-pulsar by apache.

the class BookKeeperPackagesStorageTest method testReadWriteOperationsWithSeparatedBkCluster.

@Test(timeOut = 60000)
public void testReadWriteOperationsWithSeparatedBkCluster() throws Exception {
    PackagesStorageProvider provider = PackagesStorageProvider.newProvider(BookKeeperPackagesStorageProvider.class.getName());
    DefaultPackagesStorageConfiguration configuration = new DefaultPackagesStorageConfiguration();
    // set the unavailable bk cluster with mock zookeeper path
    configuration.setProperty("metadataStoreUrl", zkUtil.getZooKeeperConnectString() + "/mock");
    configuration.setProperty("packagesReplicas", "1");
    configuration.setProperty("packagesManagementLedgerRootPath", "/ledgers");
    PackagesStorage storage1 = provider.getStorage(configuration);
    storage1.initialize();
    String mockData = "mock-data";
    ByteArrayInputStream mockDataStream = new ByteArrayInputStream(mockData.getBytes(StandardCharsets.UTF_8));
    String mockPath = "mock-path";
    // write some data to the dlog will fail
    try {
        storage1.writeAsync(mockPath, mockDataStream).get();
    } catch (Exception e) {
        String errMsg = e.getCause().getMessage();
        assertTrue(errMsg.contains("Error on allocating ledger") || errMsg.contains("Write rejected"));
    } finally {
        storage1.closeAsync().get();
    }
    // set the available bk cluster with bookkeeperMetadataServiceUri using actual zookeeper path
    String bookkeeperMetadataServiceUri = String.format("zk+null://%s/ledgers", zkUtil.getZooKeeperConnectString());
    DefaultPackagesStorageConfiguration configuration2 = new DefaultPackagesStorageConfiguration();
    configuration2.setProperty("metadataStoreUrl", zkUtil.getZooKeeperConnectString());
    configuration2.setProperty("bookkeeperMetadataServiceUri", bookkeeperMetadataServiceUri);
    configuration2.setProperty("packagesReplicas", "1");
    PackagesStorage storage2 = provider.getStorage(configuration2);
    storage2.initialize();
    String testData = "test-data";
    ByteArrayInputStream testDataStream = new ByteArrayInputStream(testData.getBytes(StandardCharsets.UTF_8));
    String testPath = "test-path";
    // write some data to the dlog will success
    try {
        storage2.writeAsync(testPath, testDataStream).get();
        // read the data from the dlog
        ByteArrayOutputStream readData = new ByteArrayOutputStream();
        storage2.readAsync(testPath, readData).get();
        String readResult = new String(readData.toByteArray(), StandardCharsets.UTF_8);
        assertEquals(testData, readResult);
    } finally {
        storage2.closeAsync().get();
    }
}