From 9ef27d4882895907758ea678f5fdaf69503d87f1 Mon Sep 17 00:00:00 2001
From: Johannes Buechele <johannes@bujo.at>
Date: Wed, 26 Jul 2023 17:28:53 +0200
Subject: [PATCH] added ipfs cluster support
---
.gitignore | 1 +
.../faircommons/ipfs_service/api/IpfsApi.java | 31 ----------
.../config/IpfsServiceConfiguration.java | 2 +
.../service/IpfsClusterService.java | 56 +++++++++++++++++++
.../service/IpfsDaemonService.java | 32 +++++++++++
.../ipfs_service/service/IpfsService.java | 38 +------------
.../service/IpfsServiceMessageHandler.java | 8 ++-
.../ipfs_service/models/IpfsAddView.java | 17 ++++++
.../src/main/resources/application.yml | 3 +
faircommons-services/pom.xml | 2 +-
10 files changed, 119 insertions(+), 71 deletions(-)
delete mode 100644 faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/api/IpfsApi.java
create mode 100644 faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsClusterService.java
create mode 100644 faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsDaemonService.java
create mode 100644 faircommons-services/ipfs-service/src/main/lombok/eu/fairkom/faircommons/ipfs_service/models/IpfsAddView.java
diff --git a/.gitignore b/.gitignore
index a6d7dc8..a4ede89 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
+faircommons-services/ipfs-service/ipfs-service.iml
# AWS User-specific
.idea/**/aws.xml
diff --git a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/api/IpfsApi.java b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/api/IpfsApi.java
deleted file mode 100644
index 04bed1b..0000000
--- a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/api/IpfsApi.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package eu.fairkom.faircommons.ipfs_service.api;
-
-import eu.fairkom.faircommons.ipfs_service.service.IpfsService;
-import org.springframework.core.io.Resource;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.multipart.MultipartFile;
-
-import java.io.IOException;
-
-@RestController
-@RequestMapping("/ipfs")
-public class IpfsApi {
- private final IpfsService ipfsService;
-
- public IpfsApi(IpfsService ipfsService) {
- this.ipfsService = ipfsService;
- }
-
- @PostMapping(value = "upload")
- public String uploadFile(@RequestParam("file") MultipartFile file) throws IOException {
- return ipfsService.saveFile(file.getInputStream());
- }
-
- @GetMapping(value = "/{hash}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
- public ResponseEntity<Resource> loadFile(@PathVariable("hash") String hash) {
- var file = ipfsService.loadFile(hash);
- return ResponseEntity.ok(file);
- }
-}
diff --git a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/config/IpfsServiceConfiguration.java b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/config/IpfsServiceConfiguration.java
index d0a1c34..a2ecf24 100644
--- a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/config/IpfsServiceConfiguration.java
+++ b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/config/IpfsServiceConfiguration.java
@@ -6,6 +6,7 @@ import eu.fairkom.faircommons.common.security.SecurityWebAutoConfiguration;
import io.ipfs.api.IPFS;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
@@ -18,6 +19,7 @@ import org.springframework.context.annotation.Import;
@Import(value = {SecurityWebAutoConfiguration.class})
public class IpfsServiceConfiguration {
@Bean
+ @ConditionalOnProperty(prefix = "ipfs", name = "service", havingValue = "daemon")
IPFS ipfsClient(@Value("${ipfs.host}") String host, @Value("${ipfs.port}") int port) {
return new IPFS(host, port);
}
diff --git a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsClusterService.java b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsClusterService.java
new file mode 100644
index 0000000..f3e10fc
--- /dev/null
+++ b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsClusterService.java
@@ -0,0 +1,56 @@
+package eu.fairkom.faircommons.ipfs_service.service;
+
+import eu.fairkom.faircommons.ipfs_service.models.IpfsAddView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.stereotype.Service;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+
+import java.io.InputStream;
+
+@Service
+@ConditionalOnProperty(prefix = "ipfs", name = "service", havingValue = "cluster")
+public class IpfsClusterService implements IpfsService {
+ private static final Logger logger = LoggerFactory.getLogger(IpfsClusterService.class);
+ private static final String UPLOAD_ENDPOINT = "/add?pin=true";
+ private final RestTemplate restTemplate;
+
+ public IpfsClusterService(RestTemplateBuilder restTemplateBuilder,
+ @Value("${ipfs.url}") String baseUrl) {
+ this.restTemplate = restTemplateBuilder.rootUri(baseUrl)
+ .build();
+ }
+
+ /**
+ * Uploads a file to IPFS.
+ * @param inputStream The InputStream of the file to upload.
+ * @return the CID of the uploaded file.
+ */
+ @Override
+ public String uploadFile(InputStream inputStream) {
+ try {
+ var body = new LinkedMultiValueMap<>();
+ body.set("file", new InputStreamResource(inputStream));
+
+ var headers = new HttpHeaders();
+ headers.setContentType(MediaType.MULTIPART_FORM_DATA);
+
+ var requestEntity = new HttpEntity<>(body, headers);
+ var response = restTemplate.postForEntity(UPLOAD_ENDPOINT, requestEntity, IpfsAddView.class);
+
+ return response.getBody().getCid();
+ } catch (RestClientException e) {
+ logger.error("Error uploading file to IPFS", e);
+ throw new RuntimeException("Failed to upload file to IPFS.", e);
+ }
+ }
+}
diff --git a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsDaemonService.java b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsDaemonService.java
new file mode 100644
index 0000000..21945b8
--- /dev/null
+++ b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsDaemonService.java
@@ -0,0 +1,32 @@
+package eu.fairkom.faircommons.ipfs_service.service;
+
+import io.ipfs.api.IPFS;
+import io.ipfs.api.NamedStreamable;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+
+@Service
+@ConditionalOnProperty(prefix = "ipfs", name = "service", havingValue = "daemon")
+public class IpfsDaemonService implements IpfsService {
+ private final IPFS ipfsClient;
+
+ public IpfsDaemonService(IPFS ipfsClient) {
+ this.ipfsClient = ipfsClient;
+ }
+
+ @Override
+ public String uploadFile(InputStream inputStream) {
+ try {
+ var inputStreamWrapper = new NamedStreamable.InputStreamWrapper(inputStream);
+ var merkleNode = ipfsClient.add(inputStreamWrapper).get(0);
+
+ return merkleNode.hash.toBase58();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsService.java b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsService.java
index 8206061..5176bea 100644
--- a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsService.java
+++ b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsService.java
@@ -1,41 +1,7 @@
package eu.fairkom.faircommons.ipfs_service.service;
-import io.ipfs.api.IPFS;
-import io.ipfs.api.NamedStreamable;
-import io.ipfs.multihash.Multihash;
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.core.io.Resource;
-import org.springframework.stereotype.Service;
-
-import java.io.IOException;
import java.io.InputStream;
-import java.io.UncheckedIOException;
-
-@Service
-public class IpfsService {
- private final IPFS ipfsClient;
-
- public IpfsService(IPFS ipfsClient) {
- this.ipfsClient = ipfsClient;
- }
-
- public String saveFile(InputStream inputStream) {
- try {
- var inputStreamWrapper = new NamedStreamable.InputStreamWrapper(inputStream);
- var merkleNode = ipfsClient.add(inputStreamWrapper).get(0);
-
- return merkleNode.hash.toBase58();
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
- public Resource loadFile(String hash) {
- try {
- var filePointer = Multihash.fromBase58(hash);
- return new InputStreamResource(ipfsClient.catStream(filePointer));
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
+public interface IpfsService {
+ String uploadFile(InputStream inputStream);
}
diff --git a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsServiceMessageHandler.java b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsServiceMessageHandler.java
index a7a1040..e7fa30b 100644
--- a/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsServiceMessageHandler.java
+++ b/faircommons-services/ipfs-service/src/main/java/eu/fairkom/faircommons/ipfs_service/service/IpfsServiceMessageHandler.java
@@ -21,7 +21,9 @@ public class IpfsServiceMessageHandler extends MessageHandler {
private final MinioService minioService;
private final IpfsService ipfsService;
- public IpfsServiceMessageHandler(RabbitTemplate rabbitTemplate, MinioService minioService, IpfsService ipfsService) {
+ public IpfsServiceMessageHandler(RabbitTemplate rabbitTemplate,
+ MinioService minioService,
+ IpfsService ipfsService) {
super(rabbitTemplate);
this.minioService = minioService;
this.ipfsService = ipfsService;
@@ -33,7 +35,7 @@ public class IpfsServiceMessageHandler extends MessageHandler {
var filename = message.getWorkFile().getFilenameWithPrefix();
try (var workFileStream = minioService.downloadFile(userId, filename)) {
- var cid = ipfsService.saveFile(workFileStream);
+ var cid = ipfsService.uploadFile(workFileStream);
var ipfsEntry = buildIpfsEntryView("", cid);
logger.info("Work File successfully saved to IPFS. Sending success event.");
@@ -47,7 +49,7 @@ public class IpfsServiceMessageHandler extends MessageHandler {
public void handleMetaDataMessage(MetaDataMessage message) {
try (var metaDataStream = new ByteArrayInputStream(message.getMetaData().getBytes(StandardCharsets.UTF_8))) {
- var cid = ipfsService.saveFile(metaDataStream);
+ var cid = ipfsService.uploadFile(metaDataStream);
var ipfsEntry = buildIpfsEntryView("", cid);
logger.info("Meta Data successfully saved to IPFS. Sending success event.");
diff --git a/faircommons-services/ipfs-service/src/main/lombok/eu/fairkom/faircommons/ipfs_service/models/IpfsAddView.java b/faircommons-services/ipfs-service/src/main/lombok/eu/fairkom/faircommons/ipfs_service/models/IpfsAddView.java
new file mode 100644
index 0000000..08c4f15
--- /dev/null
+++ b/faircommons-services/ipfs-service/src/main/lombok/eu/fairkom/faircommons/ipfs_service/models/IpfsAddView.java
@@ -0,0 +1,17 @@
+package eu.fairkom.faircommons.ipfs_service.models;
+
+import lombok.Builder;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+import java.util.List;
+
+@Value
+@Builder
+@Jacksonized
+public class IpfsAddView {
+ String name;
+ String cid;
+ int size;
+ List<String> allocations;
+}
diff --git a/faircommons-services/ipfs-service/src/main/resources/application.yml b/faircommons-services/ipfs-service/src/main/resources/application.yml
index 7c39f32..7c4df3e 100644
--- a/faircommons-services/ipfs-service/src/main/resources/application.yml
+++ b/faircommons-services/ipfs-service/src/main/resources/application.yml
@@ -20,6 +20,9 @@ minio:
bucketPrefix: fairregister-
region: eu-central-1
+ipfs:
+ service: daemon
+
security:
web:
resource:
diff --git a/faircommons-services/pom.xml b/faircommons-services/pom.xml
index 852218a..59e758f 100644
--- a/faircommons-services/pom.xml
+++ b/faircommons-services/pom.xml
@@ -6,7 +6,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
- <version>3.0.5</version>
+ <version>3.1.2</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
--
GitLab