Examples with TarImpl com.yelp.nrtsearch.server.backup.TarImpl used on opensource projects

Example 1 with TarImpl

use of com.yelp.nrtsearch.server.backup.TarImpl in project nrtsearch by Yelp.

the class RemoteStateBackendTest method setup.

@Before
public void setup() throws IOException {
    Path s3Directory = folder.newFolder("s3").toPath();
    Path archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    AmazonS3 s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(TEST_BUCKET);
    TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(s3).withShutDownThreadPools(false).build();
    ContentDownloader contentDownloader = new ContentDownloaderImpl(new TarImpl(TarImpl.CompressionMode.LZ4), transferManager, TEST_BUCKET, true);
    FileCompressAndUploader fileCompressAndUploader = new FileCompressAndUploader(new TarImpl(TarImpl.CompressionMode.LZ4), transferManager, TEST_BUCKET);
    versionManager = new VersionManager(s3, TEST_BUCKET);
    archiver = new IndexArchiver(mock(BackupDiffManager.class), fileCompressAndUploader, contentDownloader, versionManager, archiverDirectory);
}

Example 2 with TarImpl

use of com.yelp.nrtsearch.server.backup.TarImpl in project nrtsearch by Yelp.

the class RestoreStateHandlerTest method setup.

@Before
public void setup() throws IOException {
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(TarImpl.CompressionMode.LZ4));
    LuceneServerConfiguration luceneServerConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot());
    globalState = GlobalState.createState(luceneServerConfiguration);
}

Example 3 with TarImpl

use of com.yelp.nrtsearch.server.backup.TarImpl in project nrtsearch by Yelp.

the class AckedCopyTest method setUp.

public void setUp(int chunkSize, int ackEvery, int maxInFlight) throws IOException {
    // setup S3 for backup/restore
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(Tar.CompressionMode.LZ4));
    String extraConfig = String.join("\n", "FileCopyConfig:", "  ackedCopy: true", "  chunkSize: " + chunkSize, "  ackEvery: " + ackEvery, "  maxInFlight: " + maxInFlight);
    // set up primary servers
    String testIndex = "test_index";
    LuceneServerConfiguration luceneServerPrimaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.PRIMARY, folder.getRoot(), extraConfig);
    GlobalState globalStatePrimary = GlobalState.createState(luceneServerPrimaryConfiguration);
    luceneServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, false, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, globalStatePrimary.getPort(), archiver);
    replicationServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, true, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, luceneServerPrimaryConfiguration.getReplicationPort(), archiver);
    // set up secondary servers
    LuceneServerConfiguration luceneServerSecondaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.REPLICA, folder.getRoot(), extraConfig);
    GlobalState globalStateSecondary = GlobalState.createState(luceneServerSecondaryConfiguration);
    luceneServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, false, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getPort(), archiver);
    replicationServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, true, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getReplicationPort(), archiver);
}

Example 4 with TarImpl

use of com.yelp.nrtsearch.server.backup.TarImpl in project nrtsearch by Yelp.

the class BackupRestoreIndexRequestHandlerTest method setup.

@Before
public void setup() throws IOException {
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(Tar.CompressionMode.LZ4));
    grpcServer = setUpGrpcServer();
}

Example 5 with TarImpl

use of com.yelp.nrtsearch.server.backup.TarImpl in project nrtsearch by Yelp.

the class ReplicationServerTest method setUp.

@Before
public void setUp() throws IOException {
    // setup S3 for backup/restore
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(Tar.CompressionMode.LZ4));
    // set up primary servers
    String testIndex = "test_index";
    LuceneServerConfiguration luceneServerPrimaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.PRIMARY, folder.getRoot());
    GlobalState globalStatePrimary = GlobalState.createState(luceneServerPrimaryConfiguration);
    luceneServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, false, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, globalStatePrimary.getPort(), archiver);
    replicationServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, true, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, luceneServerPrimaryConfiguration.getReplicationPort(), archiver);
    // set up secondary servers
    LuceneServerConfiguration luceneServerSecondaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.REPLICA, folder.getRoot());
    GlobalState globalStateSecondary = GlobalState.createState(luceneServerSecondaryConfiguration);
    luceneServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, false, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getPort(), archiver);
    replicationServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, true, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getReplicationPort(), archiver);
}