From 3cf2425544fe917d5eaeb22f9a8e9d201d124936 Mon Sep 17 00:00:00 2001
From: Johannes Buechele <johannes@bujo.at>
Date: Wed, 4 Oct 2023 16:06:50 +0200
Subject: [PATCH] added default quota to bucket and api to set quotas
---
.../api/GlobalExceptionHandler.java | 22 ++++++++++
.../faircommons/api_service/api/MinioApi.java | 26 +++++++++++
.../api_service/api/PublicWorkApi.java | 5 ---
.../faircommons/api_service/api/WorkApi.java | 5 ---
.../api_service/api/WorkFileApi.java | 7 +--
.../src/main/resources/application.yml | 9 +++-
faircommons-services/common/pom.xml | 6 ---
.../common/minio/MinioConfiguration.java | 19 +++++---
.../common/minio/MinioProperties.java | 19 ++++++++
.../common/minio/MinioService.java | 43 ++++++++++++++-----
.../faircommons/common/minio/QuotaView.java | 5 +++
.../models/MinioConfigurationProperties.java | 14 ------
.../ResourceServerAutoConfigurer.java | 3 +-
.../src/main/resources/application.yml | 3 +-
.../src/main/resources/application.yml | 3 +-
faircommons-services/pom.xml | 13 ++++++
.../src/main/resources/application.yml | 3 +-
17 files changed, 148 insertions(+), 57 deletions(-)
create mode 100644 faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/GlobalExceptionHandler.java
create mode 100644 faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/MinioApi.java
create mode 100644 faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioProperties.java
create mode 100644 faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/QuotaView.java
delete mode 100644 faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/models/MinioConfigurationProperties.java
diff --git a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/GlobalExceptionHandler.java b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/GlobalExceptionHandler.java
new file mode 100644
index 0000000..b28ae38
--- /dev/null
+++ b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/GlobalExceptionHandler.java
@@ -0,0 +1,22 @@
+package eu.fairkom.faircommons.api_service.api;
+
+import eu.fairkom.faircommons.common.minio.MinioException;
+import jakarta.persistence.EntityNotFoundException;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+@ControllerAdvice
+public class GlobalExceptionHandler {
+ @ExceptionHandler(EntityNotFoundException.class)
+ public ResponseEntity<?> handleEntityNotFound() {
+ return ResponseEntity.notFound().build();
+ }
+
+
+ @ExceptionHandler(MinioException.class)
+ public ResponseEntity<String> handleMinioException(MinioException e) {
+ return ResponseEntity.status(503)
+ .body("Minio Service temporarily unavailable, no upload possible: ");
+ }
+}
diff --git a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/MinioApi.java b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/MinioApi.java
new file mode 100644
index 0000000..82837f1
--- /dev/null
+++ b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/MinioApi.java
@@ -0,0 +1,26 @@
+package eu.fairkom.faircommons.api_service.api;
+
+import eu.fairkom.faircommons.common.minio.MinioService;
+import eu.fairkom.faircommons.common.minio.QuotaView;
+import io.minio.admin.QuotaUnit;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+@RestController
+@RequestMapping("/minio/")
+public class MinioApi {
+ private final MinioService minioService;
+
+ public MinioApi(MinioService minioService) {
+ this.minioService = minioService;
+ }
+
+ @PutMapping("/{bucketName}/quota")
+ public ResponseEntity<Void> setQuota(
+ @PathVariable String bucketName,
+ @RequestBody QuotaView quota) {
+
+ minioService.setQuota(bucketName, quota.size(), quota.unit());
+ return ResponseEntity.noContent().build();
+ }
+}
diff --git a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/PublicWorkApi.java b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/PublicWorkApi.java
index 7940016..a95b4cb 100644
--- a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/PublicWorkApi.java
+++ b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/PublicWorkApi.java
@@ -30,9 +30,4 @@ public class PublicWorkApi {
public ResponseEntity<PublicWorkView> getWorkByGrid(@PathVariable String grid) {
return ResponseEntity.of(workService.getPublicWorkByGrid(grid));
}
-
- @ExceptionHandler(EntityNotFoundException.class)
- public ResponseEntity<?> handleEntityNotFound() {
- return ResponseEntity.notFound().build();
- }
}
diff --git a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkApi.java b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkApi.java
index de4ae6d..a99bad4 100644
--- a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkApi.java
+++ b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkApi.java
@@ -41,9 +41,4 @@ public class WorkApi {
.build(workId))
.build();
}
-
- @ExceptionHandler(EntityNotFoundException.class)
- public ResponseEntity<?> handleEntityNotFound() {
- return ResponseEntity.notFound().build();
- }
}
diff --git a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkFileApi.java b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkFileApi.java
index b2eced5..aecd968 100644
--- a/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkFileApi.java
+++ b/faircommons-services/api-service/src/main/java/eu/fairkom/faircommons/api_service/api/WorkFileApi.java
@@ -25,7 +25,7 @@ public class WorkFileApi {
@GetMapping("/upload/presigned-url")
public ResponseEntity<FileUploadView> generatePresignedUploadUrl(@RequestParam String userId,
- @RequestParam String filename) throws MinioException {
+ @RequestParam String filename) {
return ResponseEntity.ok(workFileService.generateUploadUrl(userId, filename));
}
@@ -50,9 +50,4 @@ public class WorkFileApi {
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.body(fileResponse.inputStreamResource());
}
-
- @ExceptionHandler(EntityNotFoundException.class)
- public ResponseEntity<?> handleEntityNotFound() {
- return ResponseEntity.notFound().build();
- }
}
diff --git a/faircommons-services/api-service/src/main/resources/application.yml b/faircommons-services/api-service/src/main/resources/application.yml
index fbcb101..10bfd5a 100644
--- a/faircommons-services/api-service/src/main/resources/application.yml
+++ b/faircommons-services/api-service/src/main/resources/application.yml
@@ -34,13 +34,20 @@ server:
context-path: /${spring.application.name}
minio:
- bucketPrefix: fairregister-
region: eu-central-1
+ bucket:
+ prefix: fairregister-
+
security:
web:
resource:
defaultScope: fairregister.default
+ paths:
+ - path: /minio/**
+ scopes:
+ - fairregister.admin
+
unsecured:
paths:
- /works/public/**
diff --git a/faircommons-services/common/pom.xml b/faircommons-services/common/pom.xml
index 19fc5c1..a3a6e2e 100644
--- a/faircommons-services/common/pom.xml
+++ b/faircommons-services/common/pom.xml
@@ -17,12 +17,6 @@
</properties>
<dependencies>
- <dependency>
- <groupId>io.minio</groupId>
- <artifactId>minio</artifactId>
- <version>${minio.version}</version>
- </dependency>
-
<dependency>
<groupId>org.hibernate.orm</groupId>
<artifactId>hibernate-jpamodelgen</artifactId>
diff --git a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioConfiguration.java b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioConfiguration.java
index 1841848..2d63b28 100644
--- a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioConfiguration.java
+++ b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioConfiguration.java
@@ -1,7 +1,7 @@
package eu.fairkom.faircommons.common.minio;
-import eu.fairkom.faircommons.common.minio.models.MinioConfigurationProperties;
import io.minio.MinioClient;
+import io.minio.admin.MinioAdminClient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
@@ -9,12 +9,12 @@ import org.springframework.context.annotation.Configuration;
@Configuration(proxyBeanMethods = false)
@ConditionalOnClass(MinioClient.class)
-@EnableConfigurationProperties(MinioConfigurationProperties.class)
+@EnableConfigurationProperties(MinioProperties.class)
public class MinioConfiguration {
- private final MinioConfigurationProperties properties;
+ private final MinioProperties properties;
- public MinioConfiguration(MinioConfigurationProperties minioConfigurationProperties) {
- this.properties = minioConfigurationProperties;
+ public MinioConfiguration(MinioProperties minioProperties) {
+ this.properties = minioProperties;
}
@Bean
@@ -25,4 +25,13 @@ public class MinioConfiguration {
.credentials(properties.accessKey(), properties.secretKey())
.build();
}
+
+ @Bean
+ public MinioAdminClient minioAdminClient() {
+ return MinioAdminClient.builder()
+ .endpoint(properties.url())
+ .region(properties.region())
+ .credentials(properties.accessKey(), properties.secretKey())
+ .build();
+ }
}
diff --git a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioProperties.java b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioProperties.java
new file mode 100644
index 0000000..fbeecaa
--- /dev/null
+++ b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioProperties.java
@@ -0,0 +1,19 @@
+package eu.fairkom.faircommons.common.minio;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.bind.DefaultValue;
+
+@ConfigurationProperties("minio")
+public record MinioProperties(
+ String url,
+ String accessKey,
+ String secretKey,
+ boolean secure,
+ Bucket bucket,
+ @DefaultValue("eu-central-1") String region
+) {
+ record Bucket(
+ @DefaultValue("fairregister-") String prefix,
+ @DefaultValue("200") Long quota) {
+ }
+}
diff --git a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioService.java b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioService.java
index 2b88753..aa3d435 100644
--- a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioService.java
+++ b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/MinioService.java
@@ -1,8 +1,11 @@
package eu.fairkom.faircommons.common.minio;
import io.minio.*;
+import io.minio.admin.MinioAdminClient;
+import io.minio.admin.QuotaUnit;
import io.minio.http.Method;
-import org.springframework.beans.factory.annotation.Value;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.io.InputStream;
@@ -13,13 +16,17 @@ import static eu.fairkom.faircommons.common.Constants.REGISTRATION_CERTIFICATE_N
@Service
public class MinioService {
+ private static final Logger logger = LoggerFactory.getLogger(MinioService.class);
private final MinioClient minioClient;
- @Value("${minio.bucketPrefix}")
- private String prefix;
+ private final MinioAdminClient minioAdminClient;
+ private final MinioProperties properties;
- public MinioService(MinioClient minioClient) {
+
+ public MinioService(MinioClient minioClient, MinioAdminClient minioAdminClient, MinioProperties properties) {
this.minioClient = minioClient;
+ this.minioAdminClient = minioAdminClient;
+ this.properties = properties;
}
public String generatePresignedUploadUrl(String userId, String filename) throws MinioException {
@@ -35,12 +42,13 @@ public class MinioService {
public InputStream downloadFile(String userId, String filename) throws MinioException {
try {
var downloadFileArgs = GetObjectArgs.builder()
- .bucket(prefix + userId)
+ .bucket(properties.bucket().prefix() + userId)
.object(filename)
.build();
return minioClient.getObject(downloadFileArgs);
} catch (Exception e) {
+ logger.error("Error downloading file for user: {} and file: {}", userId, filename);
throw new MinioException(e.getMessage(), e.getCause());
}
}
@@ -49,7 +57,7 @@ public class MinioService {
try {
var filename = filenamePrefix + "/" + REGISTRATION_CERTIFICATE_FOLDER + "/" + REGISTRATION_CERTIFICATE_NAME;
var uploadFileArgs = PutObjectArgs.builder()
- .bucket(prefix + userId)
+ .bucket(properties.bucket().prefix() + userId)
.object(filename)
.stream(inputStream, -1, 5242880)
.contentType("application/pdf")
@@ -57,6 +65,7 @@ public class MinioService {
minioClient.putObject(uploadFileArgs);
} catch (Exception e) {
+ logger.error("Error uploading registration certificate for user: {} with filename prefix: {}", userId, filenamePrefix);
throw new MinioException(e.getMessage(), e.getCause());
}
}
@@ -69,12 +78,13 @@ public class MinioService {
public void removeFile(String userId, String filename) throws MinioException {
try {
var removeFileArgs = RemoveObjectArgs.builder()
- .bucket(prefix + userId)
+ .bucket(properties.bucket().prefix() + userId)
.object(filename)
.build();
minioClient.removeObject(removeFileArgs);
} catch (Exception e) {
+ logger.error("Error removing file for user: {} and file: {}", userId, filename, e);
throw new MinioException(e.getMessage(), e.getCause());
}
}
@@ -83,9 +93,20 @@ public class MinioService {
try {
minioClient.makeBucket(
MakeBucketArgs.builder()
- .bucket(prefix + userId)
+ .bucket(properties.bucket().prefix() + userId)
.build());
+ setQuota(properties.bucket().prefix() + userId, properties.bucket().quota(), QuotaUnit.MB);
+ } catch (Exception e) {
+ logger.error("Error creating bucket for user: {}", userId, e);
+ throw new MinioException(e.getMessage(), e.getCause());
+ }
+ }
+
+ public void setQuota(String bucketName, Long quota, QuotaUnit unit) {
+ try {
+ minioAdminClient.setBucketQuota(bucketName, quota, unit);
} catch (Exception e) {
+ logger.error("Error setting quota for bucket: {}", bucketName, e);
throw new MinioException(e.getMessage(), e.getCause());
}
}
@@ -93,8 +114,9 @@ public class MinioService {
private boolean bucketExists(String userId) throws MinioException {
try {
return minioClient.bucketExists(BucketExistsArgs.builder()
- .bucket(prefix + userId).build());
+ .bucket(properties.bucket().prefix() + userId).build());
} catch (Exception e) {
+ logger.error("Error checking if bucket exists for user: {}", userId, e);
throw new MinioException(e.getMessage(), e.getCause());
}
}
@@ -103,12 +125,13 @@ public class MinioService {
try {
return minioClient.getPresignedObjectUrl(
GetPresignedObjectUrlArgs.builder()
- .bucket(prefix + userId)
+ .bucket(properties.bucket().prefix() + userId)
.object(filename)
.method(method)
.expiry(3, TimeUnit.HOURS)
.build());
} catch (Exception e) {
+ logger.error("Error generating presigned object URL for user: {} and file: {}", userId, filename, e);
throw new MinioException(e.getMessage(), e.getCause());
}
}
diff --git a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/QuotaView.java b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/QuotaView.java
new file mode 100644
index 0000000..91197ee
--- /dev/null
+++ b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/QuotaView.java
@@ -0,0 +1,5 @@
+package eu.fairkom.faircommons.common.minio;
+
+import io.minio.admin.QuotaUnit;
+
+public record QuotaView(long size, QuotaUnit unit) {}
\ No newline at end of file
diff --git a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/models/MinioConfigurationProperties.java b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/models/MinioConfigurationProperties.java
deleted file mode 100644
index 6372219..0000000
--- a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/minio/models/MinioConfigurationProperties.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package eu.fairkom.faircommons.common.minio.models;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-
-@ConfigurationProperties("minio")
-public record MinioConfigurationProperties(
- String url,
- String accessKey,
- String secretKey,
- boolean secure,
- String bucketPrefix,
- String region
-) {
-}
diff --git a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/security/ResourceServerAutoConfigurer.java b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/security/ResourceServerAutoConfigurer.java
index 55dacbf..be94f47 100644
--- a/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/security/ResourceServerAutoConfigurer.java
+++ b/faircommons-services/common/src/main/java/eu/fairkom/faircommons/common/security/ResourceServerAutoConfigurer.java
@@ -80,8 +80,7 @@ public class ResourceServerAutoConfigurer {
logger.info("Securing {} requests to {} with {}",
securedPath.methods(), securedPath.path(), securedPath.scopes());
authorize.requestMatchers(matcher(securedPath))
- .hasAnyAuthority(asAuthorities(securedPath.scopes()))
- .anyRequest().authenticated();
+ .hasAnyAuthority(asAuthorities(securedPath.scopes()));
});
}
diff --git a/faircommons-services/hashing-service/src/main/resources/application.yml b/faircommons-services/hashing-service/src/main/resources/application.yml
index 06d4f28..1992e49 100644
--- a/faircommons-services/hashing-service/src/main/resources/application.yml
+++ b/faircommons-services/hashing-service/src/main/resources/application.yml
@@ -16,8 +16,9 @@ server:
context-path: /${spring.application.name}
minio:
- bucketPrefix: fairregister-
region: eu-central-1
+ bucket:
+ prefix: fairregister-
security:
web:
diff --git a/faircommons-services/ipfs-service/src/main/resources/application.yml b/faircommons-services/ipfs-service/src/main/resources/application.yml
index 7c4df3e..14b86ac 100644
--- a/faircommons-services/ipfs-service/src/main/resources/application.yml
+++ b/faircommons-services/ipfs-service/src/main/resources/application.yml
@@ -17,8 +17,9 @@ server:
context-path: /${spring.application.name}
minio:
- bucketPrefix: fairregister-
region: eu-central-1
+ bucket:
+ prefix: fairregister-
ipfs:
service: daemon
diff --git a/faircommons-services/pom.xml b/faircommons-services/pom.xml
index 8bf7ff6..8ff5631 100644
--- a/faircommons-services/pom.xml
+++ b/faircommons-services/pom.xml
@@ -88,6 +88,19 @@
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>io.minio</groupId>
+ <artifactId>minio</artifactId>
+ <version>${minio.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.minio</groupId>
+ <artifactId>minio-admin</artifactId>
+ <version>${minio.version}</version>
+ </dependency>
+
+
<!-- MapStruct -->
<dependency>
<groupId>org.mapstruct</groupId>
diff --git a/faircommons-services/post-registration-service/src/main/resources/application.yml b/faircommons-services/post-registration-service/src/main/resources/application.yml
index cc67ec5..f83f4df 100644
--- a/faircommons-services/post-registration-service/src/main/resources/application.yml
+++ b/faircommons-services/post-registration-service/src/main/resources/application.yml
@@ -21,8 +21,9 @@ server:
context-path: /${spring.application.name}
minio:
- bucketPrefix: fairregister-
region: eu-central-1
+ bucket:
+ prefix: fairregister-
security:
web:
--
GitLab