Skip to content

Commit

Permalink
Merged in task/dspace-cris-2023_02_x/CST-13934 (pull request DSpace#2077
Browse files Browse the repository at this point in the history 
)

[CST-13934][DSC-1257]

Approved-by: Giuseppe Digilio
  • Loading branch information
@vins01-4science @atarix83
vins01-4science authored and atarix83 committed May 9, 2024
2 parents 3ec591e + b417007 commit 7b8e4ad
Show file tree
Hide file tree
Showing 4 changed files with 190 additions and 25 deletions.
168 changes: 152 additions & 16 deletions dspace-api/src/main/java/org/dspace/storage/bitstore/S3BitStoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,18 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Supplier;
import javax.validation.constraints.NotNull;

import com.amazonaws.AmazonClientException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.SystemDefaultDnsResolver;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
Expand Down Expand Up @@ -105,6 +109,9 @@ public class S3BitStoreService extends BaseBitStoreService {
private String awsSecretKey;
private String awsRegionName;
private boolean useRelativePath;
private Integer maxConnections;
private Integer connectionTimeout;
private String endpoint;

/**
* container for all the assets
Expand All @@ -130,23 +137,120 @@ public class S3BitStoreService extends BaseBitStoreService {
private static final ConfigurationService configurationService
= DSpaceServicesFactory.getInstance().getConfigurationService();

/**
* Utility method for generate ClientConfiguration
*
* @param maxConnections maximum number of connections for the S3 Service
* @param connectionTimeout maximum timeout for those connections
* @return ClientConfiguration with the specified parameters
*/
protected static Supplier<ClientConfiguration> getClientConfiguration(
Integer maxConnections, Integer connectionTimeout
) {
return () -> new ClientConfiguration()
.withDnsResolver(
new SystemDefaultDnsResolver()
)
.withMaxConnections(
Optional.ofNullable(maxConnections).orElse(ClientConfiguration.DEFAULT_MAX_CONNECTIONS)
)
.withConnectionTimeout(
Optional.ofNullable(connectionTimeout).orElse(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT)
);
}

/**
* Utility method for generate AmazonS3 builder
*
* @param regions wanted regions in client
* @param awsCredentials credentials of the client
* @param credentialsProvider credentials of the client
* @param clientConfiguration client connection details
* @param endpoint optional custom endpoint
* @return builder with the specified parameters
*/
protected static Supplier<AmazonS3> amazonClientBuilderBy(
@NotNull Regions regions,
@NotNull AWSCredentials awsCredentials
@NotNull Supplier<AWSStaticCredentialsProvider> credentialsProvider,
@NotNull Supplier<ClientConfiguration> clientConfiguration,
String endpoint
) {
return () -> AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(awsCredentials))
.withRegion(regions)
.build();
return () ->
withEndpointConfiguration(
AmazonS3ClientBuilder.standard()
.withCredentials(credentialsProvider.get())
//.withClientConfiguration(clientConfiguration.get())
,
regions,
endpoint
).build();
}

/**
* Utility method for generate AmazonS3 builder
*
* @param clientConfigurationSupplier client connection details
* @param endpoint optional custom endpoint
* @return builder with the specified parameters
*/
protected static Supplier<AmazonS3> amazonClientBuilderBy(
@NotNull Supplier<ClientConfiguration> clientConfigurationSupplier,
String endpoint
) {
return () ->
withEndpointConfiguration(
AmazonS3ClientBuilder.standard()
.withClientConfiguration(clientConfigurationSupplier.get()),
Regions.DEFAULT_REGION,
endpoint
).build();
}

/**
* Additional builder that enriches a given {@link AmazonS3ClientBuilder} with a custom {@link EndpointConfiguration}
* if any endpoint is set.
* <br/>
* Otherwise proceeds to set the {@link Regions} inside the builder.
*
* @param clientBuilder The builder that contains all the client details
* @param regions The region of the client to be built
* @param endpoint The custom optional endpoint to set
* @return {@link AmazonS3ClientBuilder} enriched with the given details
*/
protected static AmazonS3ClientBuilder withEndpointConfiguration(
@NotNull AmazonS3ClientBuilder clientBuilder,
@NotNull Regions regions,
String endpoint
) {
if (StringUtils.isNotBlank(endpoint)) {
clientBuilder =
clientBuilder.withEndpointConfiguration(getEndpointConfiguration(endpoint, regions));
} else {
clientBuilder = clientBuilder.withRegion(regions);
}
return clientBuilder;
}

protected static EndpointConfiguration getEndpointConfiguration(String endpoint, Regions region) {
return new EndpointConfiguration(
endpoint, region.getName()
);
}

protected static Supplier<AWSStaticCredentialsProvider> getAwsCredentialsSupplier(
String awsAccessKey, String awsSecretKey
) {
return getAwsCredentialsSupplier(
new BasicAWSCredentials(awsAccessKey, awsSecretKey)
);
}

protected static Supplier<AWSStaticCredentialsProvider> getAwsCredentialsSupplier(
AWSCredentials credentials
) {
return () -> new AWSStaticCredentialsProvider(credentials);
}


public S3BitStoreService() {}

/**
Expand All @@ -173,7 +277,7 @@ public boolean isEnabled() {
@Override
public void init() throws IOException {

if (this.isInitialized()) {
if (this.isInitialized() || !this.isEnabled()) {
return;
}

Expand All @@ -190,20 +294,27 @@ public void init() throws IOException {
}
}
// init client
s3Service = FunctionalUtils.getDefaultOrBuild(
s3Service =
FunctionalUtils.getDefaultOrBuild(
this.s3Service,
amazonClientBuilderBy(
regions,
new BasicAWSCredentials(getAwsAccessKey(), getAwsSecretKey())
)
);
regions,
getAwsCredentialsSupplier(getAwsAccessKey(), getAwsSecretKey()),
getClientConfiguration(maxConnections, connectionTimeout),
endpoint
)
);
log.warn("S3 Region set to: " + regions.getName());
} else {
log.info("Using a IAM role or aws environment credentials");
s3Service = FunctionalUtils.getDefaultOrBuild(
s3Service =
FunctionalUtils.getDefaultOrBuild(
this.s3Service,
AmazonS3ClientBuilder::defaultClient
);
amazonClientBuilderBy(
getClientConfiguration(maxConnections, connectionTimeout),
endpoint
)
);
}

// bucket name
Expand All @@ -220,7 +331,8 @@ public void init() throws IOException {
log.info("Creating new S3 Bucket: " + bucketName);
}
} catch (AmazonClientException e) {
throw new IOException(e);
log.error("Cannot locate or create the bucket: ", e);
// throw new IOException(e);
}
this.initialized = true;
log.info("AWS S3 Assetstore ready to go! bucket:" + bucketName);
Expand Down Expand Up @@ -508,6 +620,30 @@ public void setUseRelativePath(boolean useRelativePath) {
this.useRelativePath = useRelativePath;
}

public Integer getMaxConnections() {
return maxConnections;
}

public void setMaxConnections(Integer maxConnections) {
this.maxConnections = maxConnections;
}

public Integer getConnectionTimeout() {
return connectionTimeout;
}

public void setConnectionTimeout(Integer connectionTimeout) {
this.connectionTimeout = connectionTimeout;
}

public String getEndpoint() {
return endpoint;
}

public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}

/**
* Contains a command-line testing tool. Expects arguments:
* -a accessKey -s secretKey -f assetFileName
Expand Down
32 changes: 23 additions & 9 deletions dspace-api/src/test/java/org/dspace/storage/bitstore/S3BitStoreServiceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import static com.amazonaws.regions.Regions.DEFAULT_REGION;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.dspace.storage.bitstore.S3BitStoreService.CSA;
import static org.dspace.storage.bitstore.S3BitStoreService.getAwsCredentialsSupplier;
import static org.dspace.storage.bitstore.S3BitStoreService.getClientConfiguration;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
Expand All @@ -31,11 +33,8 @@
import java.util.List;
import java.util.Map;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.AnonymousAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.ObjectMetadata;
Expand Down Expand Up @@ -66,6 +65,9 @@
public class S3BitStoreServiceIT extends AbstractIntegrationTestWithDatabase {

private static final String DEFAULT_BUCKET_NAME = "dspace-asset-localhost";
public static final String S3_ENDPOINT = "http://127.0.0.1:8001";
public static final int MAX_CONNECTIONS = 5;
public static final int CONNECTION_TIMEOUT = 1000;

private S3BitStoreService s3BitStoreService;

Expand All @@ -85,9 +87,10 @@ public void setup() throws Exception {
s3Mock = S3Mock.create(8001, s3Directory.getAbsolutePath());
s3Mock.start();

amazonS3Client = createAmazonS3Client();
amazonS3Client = createAmazonS3Client(S3_ENDPOINT);

s3BitStoreService = new S3BitStoreService(amazonS3Client);
s3BitStoreService.setEnabled(true);

context.turnOffAuthorisationSystem();

Expand All @@ -106,6 +109,15 @@ public void cleanUp() throws IOException {
s3Mock.shutdown();
}

@Test
public void testBitstreamServiceNotInitializedWhenDisabled() throws IOException {
this.s3BitStoreService.setEnabled(false);

this.s3BitStoreService.init();

assertThat(this.s3BitStoreService.initialized, is(false));
}

@Test
public void testBitstreamPutAndGetWithAlreadyPresentBucket() throws IOException {

Expand Down Expand Up @@ -392,11 +404,13 @@ private byte[] generateChecksum(String content) {
}
}

private AmazonS3 createAmazonS3Client() {
return AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()))
.withEndpointConfiguration(new EndpointConfiguration("http://127.0.0.1:8001", DEFAULT_REGION.getName()))
.build();
private AmazonS3 createAmazonS3Client(String endpoint) {
return S3BitStoreService.amazonClientBuilderBy(
DEFAULT_REGION,
getAwsCredentialsSupplier(new AnonymousAWSCredentials()),
getClientConfiguration(MAX_CONNECTIONS, CONNECTION_TIMEOUT),
endpoint
).get();
}

private Item createItem() {
Expand Down
12 changes: 12 additions & 0 deletions dspace/config/modules/assetstore.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,15 @@ assetstore.s3.awsSecretKey =
# If the credentials are left empty,
# then this setting is ignored and the default AWS region will be used.
assetstore.s3.awsRegionName =

# If this property is set, overrides the default number of connections used by the S3 Client to get connected
# to remote service
assetstore.s3.maxConnections =

# If this property is set, overrides the default connection timeout (milliseconds) used by the S3 Client to get connected
# to remote service
assetstore.s3.connectionTimeout =

# If this property is set, changes the endpoint of the S3 service
assetstore.s3.endpoint =

3 changes: 3 additions & 0 deletions dspace/config/spring/api/bitstore.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@
<!-- Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!-- Optional, default is root level of bucket -->
<property name="subfolder" value="${assetstore.s3.subfolder}"/>
<property name="maxConnections" value="${assetstore.s3.maxConnections}"/>
<property name="connectionTimeout" value="${assetstore.s3.connectionTimeout}"/>
<property name="endpoint" value="${assetstore.s3.endpoint}"/>
</bean>

<!-- <bean name="localStore2 ... -->
Expand Down

0 comments on commit 7b8e4ad

Please sign in to comment.