Examples with ArchiverImpl com.yelp.nrtsearch.server.backup.ArchiverImpl used on opensource projects

Example 1 with ArchiverImpl

use of com.yelp.nrtsearch.server.backup.ArchiverImpl in project nrtsearch by Yelp.

the class RestoreStateHandlerTest method setup.

@Before
public void setup() throws IOException {
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(TarImpl.CompressionMode.LZ4));
    LuceneServerConfiguration luceneServerConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot());
    globalState = GlobalState.createState(luceneServerConfiguration);
}

Example 2 with ArchiverImpl

use of com.yelp.nrtsearch.server.backup.ArchiverImpl in project nrtsearch by Yelp.

the class AckedCopyTest method setUp.

public void setUp(int chunkSize, int ackEvery, int maxInFlight) throws IOException {
    // setup S3 for backup/restore
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(Tar.CompressionMode.LZ4));
    String extraConfig = String.join("\n", "FileCopyConfig:", "  ackedCopy: true", "  chunkSize: " + chunkSize, "  ackEvery: " + ackEvery, "  maxInFlight: " + maxInFlight);
    // set up primary servers
    String testIndex = "test_index";
    LuceneServerConfiguration luceneServerPrimaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.PRIMARY, folder.getRoot(), extraConfig);
    GlobalState globalStatePrimary = GlobalState.createState(luceneServerPrimaryConfiguration);
    luceneServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, false, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, globalStatePrimary.getPort(), archiver);
    replicationServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, true, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, luceneServerPrimaryConfiguration.getReplicationPort(), archiver);
    // set up secondary servers
    LuceneServerConfiguration luceneServerSecondaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.REPLICA, folder.getRoot(), extraConfig);
    GlobalState globalStateSecondary = GlobalState.createState(luceneServerSecondaryConfiguration);
    luceneServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, false, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getPort(), archiver);
    replicationServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, true, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getReplicationPort(), archiver);
}

Example 3 with ArchiverImpl

use of com.yelp.nrtsearch.server.backup.ArchiverImpl in project nrtsearch by Yelp.

the class BackupRestoreIndexRequestHandlerTest method setup.

@Before
public void setup() throws IOException {
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(Tar.CompressionMode.LZ4));
    grpcServer = setUpGrpcServer();
}

Example 4 with ArchiverImpl

use of com.yelp.nrtsearch.server.backup.ArchiverImpl in project nrtsearch by Yelp.

the class ReplicationServerTest method setUp.

@Before
public void setUp() throws IOException {
    // setup S3 for backup/restore
    s3Directory = folder.newFolder("s3").toPath();
    archiverDirectory = folder.newFolder("archiver").toPath();
    api = S3Mock.create(8011, s3Directory.toAbsolutePath().toString());
    api.start();
    s3 = new AmazonS3Client(new AnonymousAWSCredentials());
    s3.setEndpoint("http://127.0.0.1:8011");
    s3.createBucket(BUCKET_NAME);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(Tar.CompressionMode.LZ4));
    // set up primary servers
    String testIndex = "test_index";
    LuceneServerConfiguration luceneServerPrimaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.PRIMARY, folder.getRoot());
    GlobalState globalStatePrimary = GlobalState.createState(luceneServerPrimaryConfiguration);
    luceneServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, false, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, globalStatePrimary.getPort(), archiver);
    replicationServerPrimary = new GrpcServer(grpcCleanup, luceneServerPrimaryConfiguration, folder, true, globalStatePrimary, luceneServerPrimaryConfiguration.getIndexDir(), testIndex, luceneServerPrimaryConfiguration.getReplicationPort(), archiver);
    // set up secondary servers
    LuceneServerConfiguration luceneServerSecondaryConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.REPLICA, folder.getRoot());
    GlobalState globalStateSecondary = GlobalState.createState(luceneServerSecondaryConfiguration);
    luceneServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, false, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getPort(), archiver);
    replicationServerSecondary = new GrpcServer(grpcCleanup, luceneServerSecondaryConfiguration, folder, true, globalStateSecondary, luceneServerSecondaryConfiguration.getIndexDir(), testIndex, globalStateSecondary.getReplicationPort(), archiver);
}

Example 5 with ArchiverImpl

use of com.yelp.nrtsearch.server.backup.ArchiverImpl in project nrtsearch by Yelp.

the class FailedBackupCleanupTest method setup.

@Before
public void setup() throws IOException {
    archiverDirectory = Paths.get(folder.getRoot().toString(), "archiver");
    s3TransferStartedLatch = new CountDownLatch(1);
    s3 = new DoNothingAndWaitAmazonS3(s3TransferStartedLatch);
    archiver = new ArchiverImpl(s3, BUCKET_NAME, archiverDirectory, new TarImpl(Tar.CompressionMode.LZ4));
    luceneServerConfiguration = LuceneServerTestConfigurationFactory.getConfig(Mode.PRIMARY, folder.getRoot(), archiverDirectory);
    grpcServer = setUpGrpcServer();
}