Examples with StorageManager com.accenture.trac.common.storage.StorageManager used on opensource projects

Example 1 with StorageManager

use of com.accenture.trac.common.storage.StorageManager in project tracdap by finos.

the class TracDataService method doStartup.

@Override
protected void doStartup(Duration startupTimeout) {
    PlatformConfig platformConfig;
    DataServiceConfig dataSvcConfig;
    try {
        if (MemoryUtil.UNSAFE == null)
            throw new NullPointerException("MemoryUtil.UNSAFE == null");
    } catch (RuntimeException e) {
        log.error("Failed to set up native memory access for Apache Arrow", e);
        throw new EStartup("Failed to set up native memory access for Apache Arrow", e);
    }
    try {
        pluginManager.initRegularPlugins();
    } catch (Exception e) {
        var errorMessage = "There was a problem loading the plugins: " + e.getMessage();
        log.error(errorMessage, e);
        throw new EStartup(errorMessage, e);
    }
    try {
        log.info("Loading TRAC platform config...");
        platformConfig = configManager.loadRootConfigObject(PlatformConfig.class);
        dataSvcConfig = platformConfig.getServices().getData();
        // TODO: Config validation
        log.info("Config looks ok");
    } catch (Exception e) {
        var errorMessage = "There was a problem loading the platform config: " + e.getMessage();
        log.error(errorMessage, e);
        throw new EStartup(errorMessage, e);
    }
    try {
        var channelType = NioServerSocketChannel.class;
        var clientChannelType = NioSocketChannel.class;
        var workerThreads = Runtime.getRuntime().availableProcessors() * 2;
        workerGroup = new NioEventLoopGroup(workerThreads, new DefaultThreadFactory("data-svc"));
        bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("data-boss"));
        var execRegister = new ExecutionRegister(workerGroup);
        // TODO: Review setup of Arrow allocator, inc. interaction with Netty / Protobuf allocators
        var arrowAllocatorConfig = RootAllocator.configBuilder().allocationManagerFactory(NettyAllocationManager.FACTORY).build();
        var arrowAllocator = new RootAllocator(arrowAllocatorConfig);
        var formats = new CodecManager(pluginManager);
        var storage = new StorageManager(pluginManager);
        storage.initStorage(dataSvcConfig.getStorageMap(), formats);
        // Check default storage and format are available
        checkDefaultStorageAndFormat(storage, formats, dataSvcConfig);
        var metaClient = prepareMetadataClient(platformConfig, clientChannelType);
        var dataSvc = new DataService(dataSvcConfig, arrowAllocator, storage, formats, metaClient);
        var fileSvc = new FileService(dataSvcConfig, storage, metaClient);
        var publicApi = new TracDataApi(dataSvc, fileSvc);
        // Create the main server
        this.server = NettyServerBuilder.forPort(dataSvcConfig.getPort()).addService(publicApi).channelType(channelType).bossEventLoopGroup(bossGroup).workerEventLoopGroup(workerGroup).directExecutor().intercept(execRegister.registerExecContext()).build();
        // Good to go, let's start!
        server.start();
        log.info("Data service is listening on port {}", server.getPort());
    } catch (IOException e) {
        throw new EStartup(e.getMessage(), e);
    }
}

Example 2 with StorageManager

use of com.accenture.trac.common.storage.StorageManager in project tracdap by finos.

the class DataApiTestBase method setup.

@BeforeEach
void setup() throws Exception {
    // Create storage root dir referenced in config
    Files.createDirectory(tempDir.resolve(STORAGE_ROOT_DIR));
    var substitutions = Map.of("${TRAC_DIR}", tempDir.toString().replace("\\", "\\\\"));
    var configPath = ConfigHelpers.prepareConfig(TRAC_UNIT_CONFIG, tempDir, substitutions);
    // not yet used
    var keystoreKey = "";
    var startup = Startup.useConfigFile(TracDataService.class, tempDir, configPath.toString(), keystoreKey);
    startup.runStartupSequence();
    var plugins = startup.getPlugins();
    plugins.initRegularPlugins();
    var config = startup.getConfig();
    var platformConfig = config.loadRootConfigObject(PlatformConfig.class);
    var dataSvcConfig = platformConfig.getServices().getData();
    formats = new CodecManager(plugins);
    storage = new StorageManager(plugins);
    storage.initStorage(dataSvcConfig.getStorageMap(), formats);
    execContext = new ExecutionContext(new DefaultEventExecutor());
    var arrowAllocatorConfig = RootAllocator.configBuilder().allocationManagerFactory(NettyAllocationManager.FACTORY).build();
    var arrowAllocator = new RootAllocator(arrowAllocatorConfig);
    var dataSvcName = InProcessServerBuilder.generateName();
    workerGroup = new NioEventLoopGroup(6, new DefaultThreadFactory("data-svc"));
    var execRegister = new ExecutionRegister(workerGroup);
    var dataSvcClientChannelBuilder = NettyChannelBuilder.forAddress("localhost", METADATA_SVC_PORT).channelType(NioSocketChannel.class).eventLoopGroup(workerGroup).directExecutor().usePlaintext();
    dataSvcClientChannel = EventLoopChannel.wrapChannel(dataSvcClientChannelBuilder, workerGroup);
    var metaApi = TrustedMetadataApiGrpc.newFutureStub(dataSvcClientChannel);
    var dataRwSvc = new DataService(dataSvcConfig, arrowAllocator, storage, formats, metaApi);
    var fileRwSvc = new FileService(dataSvcConfig, storage, metaApi);
    var publicApiImpl = new TracDataApi(dataRwSvc, fileRwSvc);
    dataService = InProcessServerBuilder.forName(dataSvcName).addService(publicApiImpl).executor(workerGroup).intercept(execRegister.registerExecContext()).build().start();
    // Create a client channel and register for automatic graceful shutdown.
    dataClientChannel = InProcessChannelBuilder.forName(dataSvcName).directExecutor().build();
    dataClient = TracDataApiGrpc.newStub(dataClientChannel);
}