From b49ee1513dab4ef0b219388780bd5795169a4538 Mon Sep 17 00:00:00 2001
From: Simon Li <spli@dundee.ac.uk>
Date: Thu, 31 Jan 2019 15:53:20 +0000
Subject: [PATCH 1/7] Add S3ClientService wrapper for Minio

---
 .../loci/common/services/S3ClientService.java |  88 ++++++++++++
 .../services/S3ClientServiceException.java    |  67 +++++++++
 .../common/services/S3ClientServiceImpl.java  | 132 ++++++++++++++++++
 .../loci/common/services/S3ClientStat.java    |  45 ++++++
 .../common/utests/S3ClientServiceTest.java    | 129 +++++++++++++++++
 5 files changed, 461 insertions(+)
 create mode 100644 src/main/java/loci/common/services/S3ClientService.java
 create mode 100644 src/main/java/loci/common/services/S3ClientServiceException.java
 create mode 100644 src/main/java/loci/common/services/S3ClientServiceImpl.java
 create mode 100644 src/main/java/loci/common/services/S3ClientStat.java
 create mode 100644 src/test/java/loci/common/utests/S3ClientServiceTest.java

diff --git a/src/main/java/loci/common/services/S3ClientService.java b/src/main/java/loci/common/services/S3ClientService.java
new file mode 100644
index 00000000..3e902387
--- /dev/null
+++ b/src/main/java/loci/common/services/S3ClientService.java
@@ -0,0 +1,88 @@
+/*
+ * #%L
+ * S3 client service
+ * %%
+ * Copyright (C) 2019 Open Microscopy Environment:
+ *   - Board of Regents of the University of Wisconsin-Madison
+ *   - Glencoe Software, Inc.
+ *   - University of Dundee
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+
+package loci.common.services;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An S3 client
+ */
+public interface S3ClientService extends Service {
+
+  /**
+   * Initialise the S3 client
+   * @param server servername
+   * @param port port
+   * @param accessKey access key
+   * @param secretKey secret key
+   * @param appName user agent application name
+   * @param appVersion user agent application version
+   */
+  void initialise(String server, int port, String accessKey, String secretKey,
+                  String appName, String appVersion)
+      throws S3ClientServiceException;
+
+  /**
+   * Check whether a bucket exists
+   * @param bucket Bucket name
+   * @return true if bucket exists
+   */
+  boolean bucketExists(String bucket) throws S3ClientServiceException, IOException;
+
+  /**
+   * Stat the object
+   * @param bucket Bucket name
+   * @param object Object path
+   * @return S3ClientStat object
+   */
+  S3ClientStat statObject(String bucket, String object) throws S3ClientServiceException, IOException;
+
+  /**
+   * Read an object
+   * @param bucket Bucket name
+   * @param object Object path
+   * @param offset Start reading at this offset
+   * @return InputStream to the object
+   */
+  InputStream getObject(String bucket, String object, long offset) throws S3ClientServiceException, IOException;
+
+  /**
+   * Download an object
+   * @param bucket Bucket name
+   * @param object Object path
+   * @param filename Destination file
+   */
+  void getObject(String bucket, String object, String filename) throws S3ClientServiceException, IOException;
+
+}
diff --git a/src/main/java/loci/common/services/S3ClientServiceException.java b/src/main/java/loci/common/services/S3ClientServiceException.java
new file mode 100644
index 00000000..6f54d770
--- /dev/null
+++ b/src/main/java/loci/common/services/S3ClientServiceException.java
@@ -0,0 +1,67 @@
+/*
+ * #%L
+ * S3 client stat object
+ * %%
+ * Copyright (C) 2019 Open Microscopy Environment:
+ *   - Board of Regents of the University of Wisconsin-Madison
+ *   - Glencoe Software, Inc.
+ *   - University of Dundee
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+
+package loci.common.services;
+
+/**
+ * Exception thrown when internal error specific to the S3 client is raised
+ */
+public class S3ClientServiceException extends ServiceException
+{
+  /**
+   * Default constructor.
+   * @param message Error message.
+   */
+  public S3ClientServiceException(String message)
+  {
+    super(message);
+  }
+
+  /**
+   * Default constructor.
+   * @param message Error message. 
+   * @param cause Upstream exception.
+   */
+  public S3ClientServiceException(String message, Throwable cause)
+  {
+    super(message, cause);
+  }
+
+  /**
+   * Default constructor.
+   * @param cause Upstream exception.
+   */
+  public S3ClientServiceException(Throwable cause)
+  {
+    super(cause);
+  }
+}
diff --git a/src/main/java/loci/common/services/S3ClientServiceImpl.java b/src/main/java/loci/common/services/S3ClientServiceImpl.java
new file mode 100644
index 00000000..387afd60
--- /dev/null
+++ b/src/main/java/loci/common/services/S3ClientServiceImpl.java
@@ -0,0 +1,132 @@
+/*
+ * #%L
+ * S3 client service
+ * %%
+ * Copyright (C) 2019 Open Microscopy Environment:
+ *   - Board of Regents of the University of Wisconsin-Madison
+ *   - Glencoe Software, Inc.
+ *   - University of Dundee
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+
+package loci.common.services;
+import io.minio.MinioClient;
+import io.minio.errors.MinioException;
+import io.minio.ObjectStat;
+import org.xmlpull.v1.XmlPullParserException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * An S3 client
+ */
+public class S3ClientServiceImpl extends AbstractService implements S3ClientService {
+
+  /**
+   * Minio client
+   */
+  private MinioClient s3Client;
+
+  /**
+   * Default constructor.
+   */
+  public S3ClientServiceImpl() {
+    checkClassDependency(io.minio.MinioClient.class);
+  }
+
+  // -- S3ClientService methods
+
+  @Override
+  public void initialise(String server, int port, String accessKey, String secretKey,
+                         String appName, String appVersion)
+      throws S3ClientServiceException {
+    try {
+      s3Client = new MinioClient(server, port, accessKey, secretKey);
+      s3Client.setAppInfo(appName, appVersion);
+    } catch (MinioException e) {
+      throw new S3ClientServiceException(e);
+    }
+  }
+
+  @Override
+  public boolean bucketExists(String bucket) throws S3ClientServiceException, IOException {
+    try {
+      return s3Client.bucketExists(bucket);
+    }
+    catch (
+        MinioException |
+        InvalidKeyException |
+        NoSuchAlgorithmException |
+        XmlPullParserException e) {
+      throw new S3ClientServiceException(e);
+    }
+  }
+
+  @Override
+  public S3ClientStat statObject(String bucket, String object) throws S3ClientServiceException, IOException {
+    try {
+      ObjectStat mcstat = s3Client.statObject(bucket, object);
+      return new S3ClientStat(mcstat.length());
+    }
+    catch (
+        MinioException |
+        InvalidKeyException |
+        NoSuchAlgorithmException |
+        XmlPullParserException e) {
+      throw new S3ClientServiceException(e);
+    }
+  }
+
+  @Override
+  public InputStream getObject(String bucket, String object, long offset) throws S3ClientServiceException, IOException {
+    try {
+      return s3Client.getObject(bucket, object, offset);
+    }
+    catch (
+        InvalidKeyException |
+        MinioException |
+        NoSuchAlgorithmException |
+        XmlPullParserException e) {
+      throw new S3ClientServiceException(e);
+    }
+  }
+
+  @Override
+  public void getObject(String bucket, String object, String filename) throws S3ClientServiceException, IOException {
+    try {
+      s3Client.getObject(bucket, object, filename);
+    }
+    catch (
+        InvalidKeyException |
+            MinioException |
+            NoSuchAlgorithmException |
+            XmlPullParserException e) {
+      throw new S3ClientServiceException(e);
+    }
+  }
+
+}
diff --git a/src/main/java/loci/common/services/S3ClientStat.java b/src/main/java/loci/common/services/S3ClientStat.java
new file mode 100644
index 00000000..5ac2a07b
--- /dev/null
+++ b/src/main/java/loci/common/services/S3ClientStat.java
@@ -0,0 +1,45 @@
+/*
+ * #%L
+ * S3 client stat object
+ * %%
+ * Copyright (C) 2019 Open Microscopy Environment:
+ *   - Board of Regents of the University of Wisconsin-Madison
+ *   - Glencoe Software, Inc.
+ *   - University of Dundee
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+
+package loci.common.services;
+
+public class S3ClientStat {
+  S3ClientStat(long length) {
+    this.length = length;
+  }
+
+  private final long length;
+
+  public long length() {
+    return this.length;
+  };
+}
\ No newline at end of file
diff --git a/src/test/java/loci/common/utests/S3ClientServiceTest.java b/src/test/java/loci/common/utests/S3ClientServiceTest.java
new file mode 100644
index 00000000..c80c5205
--- /dev/null
+++ b/src/test/java/loci/common/utests/S3ClientServiceTest.java
@@ -0,0 +1,129 @@
+/*
+ * #%L
+ * Common package for I/O and related utilities
+ * %%
+ * Copyright (C) 2005 - 2016 Open Microscopy Environment:
+ *   - Board of Regents of the University of Wisconsin-Madison
+ *   - Glencoe Software, Inc.
+ *   - University of Dundee
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+
+package loci.common.utests;
+
+import loci.common.services.S3ClientService;
+import loci.common.services.S3ClientServiceException;
+import loci.common.services.S3ClientServiceImpl;
+import loci.common.services.S3ClientStat;
+import org.testng.SkipException;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+/**
+ * Unit tests for the loci.common.service.S3ClientServiceImpl class.
+ *
+ * @see loci.common.URLHandle
+ */
+@Test(groups="readTests")
+public class S3ClientServiceTest {
+
+  // -- Fields --
+
+  private static boolean runS3RemoteTests;
+  private Path TEMPDIR;
+  private S3ClientService s3;
+
+  // -- Setup methods --
+
+  @BeforeClass
+  public void setup() throws IOException {
+    TEMPDIR = Files.createTempDirectory("S3ClientServiceTest-");
+    TEMPDIR.toFile().deleteOnExit();
+
+    runS3RemoteTests = TestUtilities.getPropValueInt("testng.runS3RemoteTests") > 0;
+
+    if (!runS3RemoteTests) {
+      System.err.println("WARNING: S3 tests are disabled!");
+    }
+  }
+
+  @BeforeMethod
+  public void setupMethod() throws S3ClientServiceException, SkipException {
+    if (!runS3RemoteTests) {
+      throw new SkipException("S3 tests are disabled");
+    }
+    s3 = new S3ClientServiceImpl();
+    s3.initialise("http://localhost", 31836, null, null, "S3ClientServiceTest", "0.0.0");
+  }
+
+  // -- Test methods --
+
+  @Test
+  public void testBucketExists() throws S3ClientServiceException, IOException {
+    assertTrue(s3.bucketExists("bioformats.test.public"));
+  }
+
+  @Test(expectedExceptions = { S3ClientServiceException.class })
+  public void testBucketNotExists() throws S3ClientServiceException, IOException {
+    s3.bucketExists("this.bucket.does.not.exist");
+  }
+
+  @Test
+  public void testStatObject() throws S3ClientServiceException, IOException {
+    S3ClientStat stat = s3.statObject("bioformats.test.public", "2MBfile.txt");
+    assertEquals(2097152, stat.length());
+  }
+
+  @Test
+  public void testGetObject() throws S3ClientServiceException, IOException {
+    InputStream in = s3.getObject("bioformats.test.public", "2MBfile.txt", 1380896);
+    String line = new BufferedReader(new InputStreamReader(in)).readLine();
+    assertEquals(".                         43154", line);
+    in.close();
+  }
+
+  @Test
+  public void testGetObjectFile() throws S3ClientServiceException, IOException {
+    final String filepath = TEMPDIR + "/2MBfile.txt";
+    s3.getObject("bioformats.test.public", "2MBfile.txt", filepath);
+    Path path = Paths.get(filepath);
+    assertEquals(2097152, Files.size(path));
+    String[] lines = Files.newBufferedReader(path).lines().toArray(String[]::new);
+    assertEquals(".                             1", lines[0]);
+    assertEquals(".                         65536", lines[lines.length - 1]);
+  }
+
+}
\ No newline at end of file

From 12174cb668f3f38768f80e5c26bcf9728abfe6fb Mon Sep 17 00:00:00 2001
From: Simon Li <spli@dundee.ac.uk>
Date: Thu, 31 Jan 2019 16:48:35 +0000
Subject: [PATCH 2/7] Register S3ClientService

---
 src/main/java/loci/common/services/services.properties   | 2 ++
 .../java/loci/common/utests/S3ClientServiceTest.java     | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/src/main/java/loci/common/services/services.properties b/src/main/java/loci/common/services/services.properties
index 18f1deb8..ee701222 100644
--- a/src/main/java/loci/common/services/services.properties
+++ b/src/main/java/loci/common/services/services.properties
@@ -58,6 +58,8 @@ loci.formats.services.JPEGTurboService=loci.formats.services.JPEGTurboServiceImp
 loci.formats.services.WlzService=loci.formats.services.WlzServiceImpl
 loci.formats.services.EXIFService=loci.formats.services.EXIFServiceImpl
 loci.formats.services.JPEGXRService=loci.formats.services.JPEGXRServiceImpl
+# Minio S3 client service
+loci.common.services.S3ClientService=loci.common.services.S3ClientServiceImpl
 # OME Codecs JAI Image I/O service
 ome.codecs.services.JAIIIOService=ome.codecs.services.JAIIIOServiceImpl
 # OME Codecs LuraWave decoder service (depends on stubs.jar for compilation)
diff --git a/src/test/java/loci/common/utests/S3ClientServiceTest.java b/src/test/java/loci/common/utests/S3ClientServiceTest.java
index c80c5205..6d806d99 100644
--- a/src/test/java/loci/common/utests/S3ClientServiceTest.java
+++ b/src/test/java/loci/common/utests/S3ClientServiceTest.java
@@ -32,10 +32,12 @@
 
 package loci.common.utests;
 
+import loci.common.services.DependencyException;
 import loci.common.services.S3ClientService;
 import loci.common.services.S3ClientServiceException;
 import loci.common.services.S3ClientServiceImpl;
 import loci.common.services.S3ClientStat;
+import loci.common.services.ServiceFactory;
 import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
@@ -91,6 +93,13 @@ public void setupMethod() throws S3ClientServiceException, SkipException {
 
   // -- Test methods --
 
+  @Test
+  public void testServiceLookup() throws DependencyException {
+    ServiceFactory factory = new ServiceFactory();
+    S3ClientService s3client = factory.getInstance(S3ClientService.class);
+    assertTrue(S3ClientServiceImpl.class.isInstance(s3client));
+  }
+
   @Test
   public void testBucketExists() throws S3ClientServiceException, IOException {
     assertTrue(s3.bucketExists("bioformats.test.public"));

From 3a597777f66b9c5e2dbed5bedd4a08775b41f515 Mon Sep 17 00:00:00 2001
From: Simon Li <spli@dundee.ac.uk>
Date: Fri, 1 Feb 2019 13:26:15 +0000
Subject: [PATCH 3/7] S3Handle uses S3ClientService

---
 src/main/java/loci/common/S3Handle.java | 61 ++++++++++---------------
 1 file changed, 24 insertions(+), 37 deletions(-)

diff --git a/src/main/java/loci/common/S3Handle.java b/src/main/java/loci/common/S3Handle.java
index 9862007e..443f9d03 100644
--- a/src/main/java/loci/common/S3Handle.java
+++ b/src/main/java/loci/common/S3Handle.java
@@ -40,15 +40,14 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
 import java.util.regex.Pattern;
 
-import io.minio.MinioClient;
-import io.minio.errors.MinioException;
-import io.minio.ObjectStat;
-import org.xmlpull.v1.XmlPullParserException;
+import loci.common.services.DependencyException;
+import loci.common.services.S3ClientService;
+import loci.common.services.S3ClientServiceException;
+import loci.common.services.S3ClientStat;
 
+import loci.common.services.ServiceFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -103,11 +102,11 @@ class DelayedObjectNotFound extends IOException {
   /** remaining path, or key, for this accessed resource */
   private final String path;
 
-  /** Minio client */
-  private MinioClient s3Client;
+  /** S3 client */
+  private S3ClientService s3Client;
 
   /** Remote file stat */
-  private ObjectStat stat;
+  private S3ClientStat stat;
 
   /** Is this a directory (currently only buckets are considered directories */
   private boolean isBucket;
@@ -228,11 +227,7 @@ else if (scheme.startsWith("s3+")) {
       try {
         this.initialize();
       }
-      catch (
-        MinioException |
-        InvalidKeyException |
-        NoSuchAlgorithmException |
-        XmlPullParserException e) {
+      catch (S3ClientServiceException e) {
         this.objectNotFound = e;
         LOGGER.debug("Object not found: [{}] {}", this, e);
       }
@@ -245,32 +240,33 @@ else if (scheme.startsWith("s3+")) {
    * @throws IOException if there was an error connecting to the server
    */
   protected void connect() throws IOException {
+    final String appName = "Bio-Formats";
+    // TODO: Replace "dev" with a version
+    final String appVersion = "dev";
     try {
-      s3Client = new MinioClient(server, port, accessKey, secretKey);
-      // TODO: Replace "dev" with a version
-      s3Client.setAppInfo("Bio-Formats", "dev");
+      ServiceFactory factory = new ServiceFactory();
+      s3Client = factory.getInstance(S3ClientService.class);
+      s3Client.initialise(server, port, accessKey, secretKey, appName, appVersion);
     }
-    catch (MinioException e) {
+    catch (S3ClientServiceException e) {
       throw new IOException(String.format(
               "Failed to connect: %s", this), e);
     }
+    catch (DependencyException e) {
+      throw new IOException(String.format(
+          "S3 requires additional dependencies: %s", this), e);
+    }
     LOGGER.trace("connected: server:{} port:{}", server, port);
   }
 
   /**
    * Check bucket or object exists
    * @throws IOException if unable to get the object
-   * @throws MinioException if unable to get the object
-   * @throws InvalidKeyException if unable to get the object
-   * @throws NoSuchAlgorithmException if unable to get the object
-   * @throws XmlPullParserException if unable to get the object
+   * @throws S3ClientServiceException if unable to get the object
    */
   protected void initialize() throws
       IOException,
-      MinioException,
-      InvalidKeyException,
-      NoSuchAlgorithmException,
-      XmlPullParserException {
+      S3ClientServiceException {
     if (path == null) {
       isBucket = s3Client.bucketExists(bucket);
     }
@@ -351,12 +347,7 @@ protected void downloadObject(Path destination) throws HandleException, IOExcept
       Files.createDirectories(destination.getParent());
       s3Client.getObject(bucket, path, destination.toString());
     }
-    catch (
-      IOException |
-      InvalidKeyException |
-      MinioException |
-      NoSuchAlgorithmException |
-      XmlPullParserException e) {
+    catch (S3ClientServiceException e) {
         throw new HandleException("Download failed " + toString(), e);
       }
   }
@@ -437,11 +428,7 @@ protected void resetStream(long offset) throws IOException {
       fp = offset;
       mark = offset;
       }
-      catch (
-        InvalidKeyException |
-        MinioException |
-        NoSuchAlgorithmException |
-        XmlPullParserException e) {
+      catch (S3ClientServiceException e) {
         throw new IOException(String.format(
               "failed to load s3: %s\n\t%s", uri, this), e);
     }

From 9f2b5dc2ce7a30b0ecd9c4dda362db19193d6adc Mon Sep 17 00:00:00 2001
From: Simon Li <spli@dundee.ac.uk>
Date: Tue, 5 Feb 2019 10:13:16 +0000
Subject: [PATCH 4/7] s3: use "initialize" spelling

---
 src/main/java/loci/common/S3Handle.java                     | 2 +-
 src/main/java/loci/common/services/S3ClientService.java     | 2 +-
 src/main/java/loci/common/services/S3ClientServiceImpl.java | 2 +-
 src/test/java/loci/common/utests/S3ClientServiceTest.java   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/loci/common/S3Handle.java b/src/main/java/loci/common/S3Handle.java
index 443f9d03..5ca3c3dc 100644
--- a/src/main/java/loci/common/S3Handle.java
+++ b/src/main/java/loci/common/S3Handle.java
@@ -246,7 +246,7 @@ protected void connect() throws IOException {
     try {
       ServiceFactory factory = new ServiceFactory();
       s3Client = factory.getInstance(S3ClientService.class);
-      s3Client.initialise(server, port, accessKey, secretKey, appName, appVersion);
+      s3Client.initialize(server, port, accessKey, secretKey, appName, appVersion);
     }
     catch (S3ClientServiceException e) {
       throw new IOException(String.format(
diff --git a/src/main/java/loci/common/services/S3ClientService.java b/src/main/java/loci/common/services/S3ClientService.java
index 3e902387..faa20b8c 100644
--- a/src/main/java/loci/common/services/S3ClientService.java
+++ b/src/main/java/loci/common/services/S3ClientService.java
@@ -49,7 +49,7 @@ public interface S3ClientService extends Service {
    * @param appName user agent application name
    * @param appVersion user agent application version
    */
-  void initialise(String server, int port, String accessKey, String secretKey,
+  void initialize(String server, int port, String accessKey, String secretKey,
                   String appName, String appVersion)
       throws S3ClientServiceException;
 
diff --git a/src/main/java/loci/common/services/S3ClientServiceImpl.java b/src/main/java/loci/common/services/S3ClientServiceImpl.java
index 387afd60..9a44d94c 100644
--- a/src/main/java/loci/common/services/S3ClientServiceImpl.java
+++ b/src/main/java/loci/common/services/S3ClientServiceImpl.java
@@ -61,7 +61,7 @@ public S3ClientServiceImpl() {
   // -- S3ClientService methods
 
   @Override
-  public void initialise(String server, int port, String accessKey, String secretKey,
+  public void initialize(String server, int port, String accessKey, String secretKey,
                          String appName, String appVersion)
       throws S3ClientServiceException {
     try {
diff --git a/src/test/java/loci/common/utests/S3ClientServiceTest.java b/src/test/java/loci/common/utests/S3ClientServiceTest.java
index 6d806d99..06d88047 100644
--- a/src/test/java/loci/common/utests/S3ClientServiceTest.java
+++ b/src/test/java/loci/common/utests/S3ClientServiceTest.java
@@ -88,7 +88,7 @@ public void setupMethod() throws S3ClientServiceException, SkipException {
       throw new SkipException("S3 tests are disabled");
     }
     s3 = new S3ClientServiceImpl();
-    s3.initialise("http://localhost", 31836, null, null, "S3ClientServiceTest", "0.0.0");
+    s3.initialize("http://localhost", 31836, null, null, "S3ClientServiceTest", "0.0.0");
   }
 
   // -- Test methods --

From 62b65bde3e7437478df93f0c620262127866d942 Mon Sep 17 00:00:00 2001
From: Simon Li <spli@dundee.ac.uk>
Date: Tue, 5 Feb 2019 10:14:06 +0000
Subject: [PATCH 5/7] s3: cleanups, more javadocs

---
 src/main/java/loci/common/S3Handle.java            |  2 +-
 .../common/services/S3ClientServiceException.java  |  6 +++---
 .../loci/common/services/S3ClientServiceImpl.java  |  6 +++---
 .../java/loci/common/services/S3ClientStat.java    | 14 ++++++++++++++
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/main/java/loci/common/S3Handle.java b/src/main/java/loci/common/S3Handle.java
index 5ca3c3dc..6a323c69 100644
--- a/src/main/java/loci/common/S3Handle.java
+++ b/src/main/java/loci/common/S3Handle.java
@@ -343,8 +343,8 @@ protected void downloadObject(Path destination) throws HandleException, IOExcept
     if (path == null) {
       throw new HandleException("Download path=null not allowed");
     }
+    Files.createDirectories(destination.getParent());
     try {
-      Files.createDirectories(destination.getParent());
       s3Client.getObject(bucket, path, destination.toString());
     }
     catch (S3ClientServiceException e) {
diff --git a/src/main/java/loci/common/services/S3ClientServiceException.java b/src/main/java/loci/common/services/S3ClientServiceException.java
index 6f54d770..2c2d4a80 100644
--- a/src/main/java/loci/common/services/S3ClientServiceException.java
+++ b/src/main/java/loci/common/services/S3ClientServiceException.java
@@ -38,7 +38,7 @@
 public class S3ClientServiceException extends ServiceException
 {
   /**
-   * Default constructor.
+   * Constructor.
    * @param message Error message.
    */
   public S3ClientServiceException(String message)
@@ -47,7 +47,7 @@ public S3ClientServiceException(String message)
   }
 
   /**
-   * Default constructor.
+   * Constructor.
    * @param message Error message. 
    * @param cause Upstream exception.
    */
@@ -57,7 +57,7 @@ public S3ClientServiceException(String message, Throwable cause)
   }
 
   /**
-   * Default constructor.
+   * Constructor.
    * @param cause Upstream exception.
    */
   public S3ClientServiceException(Throwable cause)
diff --git a/src/main/java/loci/common/services/S3ClientServiceImpl.java b/src/main/java/loci/common/services/S3ClientServiceImpl.java
index 9a44d94c..3b12b04c 100644
--- a/src/main/java/loci/common/services/S3ClientServiceImpl.java
+++ b/src/main/java/loci/common/services/S3ClientServiceImpl.java
@@ -122,9 +122,9 @@ public void getObject(String bucket, String object, String filename) throws S3Cl
     }
     catch (
         InvalidKeyException |
-            MinioException |
-            NoSuchAlgorithmException |
-            XmlPullParserException e) {
+        MinioException |
+        NoSuchAlgorithmException |
+        XmlPullParserException e) {
       throw new S3ClientServiceException(e);
     }
   }
diff --git a/src/main/java/loci/common/services/S3ClientStat.java b/src/main/java/loci/common/services/S3ClientStat.java
index 5ac2a07b..feceb1b4 100644
--- a/src/main/java/loci/common/services/S3ClientStat.java
+++ b/src/main/java/loci/common/services/S3ClientStat.java
@@ -32,13 +32,27 @@
 
 package loci.common.services;
 
+/**
+ * Object stat (attributes) information
+ */
 public class S3ClientStat {
+  /**
+   * Create stat object storing length of object
+   * @param length
+   */
   S3ClientStat(long length) {
     this.length = length;
   }
 
+  /**
+   * Length (size) of object
+   */
   private final long length;
 
+  /**
+   * Get length (size) of object
+   * @return length
+   */
   public long length() {
     return this.length;
   };

From 4bfb0e36a5f6c4aee22cfa8d3f9967403f41b8d4 Mon Sep 17 00:00:00 2001
From: Simon Li <spli@dundee.ac.uk>
Date: Tue, 5 Feb 2019 10:26:45 +0000
Subject: [PATCH 6/7] S3: Use slf4j to print disabled test warning

---
 src/test/java/loci/common/utests/LocationTest.java        | 7 +++++--
 src/test/java/loci/common/utests/S3ClientServiceTest.java | 6 +++++-
 src/test/java/loci/common/utests/S3HandleTest.java        | 6 +++++-
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/test/java/loci/common/utests/LocationTest.java b/src/test/java/loci/common/utests/LocationTest.java
index dc6c5371..02a24ea8 100644
--- a/src/test/java/loci/common/utests/LocationTest.java
+++ b/src/test/java/loci/common/utests/LocationTest.java
@@ -43,6 +43,8 @@
 
 import loci.common.Location;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -55,6 +57,7 @@
 public class LocationTest {
 
   private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows");
+  private static final Logger LOGGER = LoggerFactory.getLogger(LocationTest.class);
 
   // -- Fields --
 
@@ -202,10 +205,10 @@ public void checkProperties() throws IOException {
     runS3RemoteTests = TestUtilities.getPropValueInt("testng.runS3RemoteTests") > 0;
 
     if (!runHttpRemoteTests) {
-      System.err.println("WARNING: HTTP tests are disabled!");
+      LOGGER.warn("HTTP tests are disabled!");
     }
     if (!runS3RemoteTests) {
-      System.err.println("WARNING: S3 tests are disabled!");
+      LOGGER.warn("S3 tests are disabled!");
     }
   }
 
diff --git a/src/test/java/loci/common/utests/S3ClientServiceTest.java b/src/test/java/loci/common/utests/S3ClientServiceTest.java
index 06d88047..fa0bb657 100644
--- a/src/test/java/loci/common/utests/S3ClientServiceTest.java
+++ b/src/test/java/loci/common/utests/S3ClientServiceTest.java
@@ -38,6 +38,8 @@
 import loci.common.services.S3ClientServiceImpl;
 import loci.common.services.S3ClientStat;
 import loci.common.services.ServiceFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
@@ -68,6 +70,8 @@ public class S3ClientServiceTest {
   private Path TEMPDIR;
   private S3ClientService s3;
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(S3ClientServiceTest.class);
+
   // -- Setup methods --
 
   @BeforeClass
@@ -78,7 +82,7 @@ public void setup() throws IOException {
     runS3RemoteTests = TestUtilities.getPropValueInt("testng.runS3RemoteTests") > 0;
 
     if (!runS3RemoteTests) {
-      System.err.println("WARNING: S3 tests are disabled!");
+      LOGGER.warn("S3 tests are disabled!");
     }
   }
 
diff --git a/src/test/java/loci/common/utests/S3HandleTest.java b/src/test/java/loci/common/utests/S3HandleTest.java
index 8b86428e..fc49416f 100644
--- a/src/test/java/loci/common/utests/S3HandleTest.java
+++ b/src/test/java/loci/common/utests/S3HandleTest.java
@@ -34,6 +34,8 @@
 
 import loci.common.S3Handle;
 import loci.common.StreamHandle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -60,6 +62,8 @@ public class S3HandleTest {
   private static boolean runS3RemoteTests;
   private Path TEMPDIR;
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(S3HandleTest.class);
+
   private static final String s3public = "s3+http://localhost:31836";
   private static final String s3private = "s3+http://accesskey:secretkey@localhost:31836";
 
@@ -73,7 +77,7 @@ public void setup() throws IOException {
     runS3RemoteTests = TestUtilities.getPropValueInt("testng.runS3RemoteTests") > 0;
 
     if (!runS3RemoteTests) {
-      System.err.println("WARNING: S3 tests are disabled!");
+      LOGGER.warn("S3 tests are disabled!");
     }
   }
 

From 03412431ca2d72f84e0c42e5e23945dfa71a981a Mon Sep 17 00:00:00 2001
From: Simon Li <spli@dundee.ac.uk>
Date: Tue, 5 Feb 2019 10:27:07 +0000
Subject: [PATCH 7/7] S3: Add @throws to silence javadocs warnings

---
 src/main/java/loci/common/services/S3ClientService.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/loci/common/services/S3ClientService.java b/src/main/java/loci/common/services/S3ClientService.java
index faa20b8c..55e046e0 100644
--- a/src/main/java/loci/common/services/S3ClientService.java
+++ b/src/main/java/loci/common/services/S3ClientService.java
@@ -48,6 +48,7 @@ public interface S3ClientService extends Service {
    * @param secretKey secret key
    * @param appName user agent application name
    * @param appVersion user agent application version
+   * @throws S3ClientServiceException if an S3 error occurred
    */
   void initialize(String server, int port, String accessKey, String secretKey,
                   String appName, String appVersion)
@@ -57,6 +58,8 @@ void initialize(String server, int port, String accessKey, String secretKey,
    * Check whether a bucket exists
    * @param bucket Bucket name
    * @return true if bucket exists
+   * @throws S3ClientServiceException if an S3 error occurred
+   * @throws IOException if an S3 error occurred
    */
   boolean bucketExists(String bucket) throws S3ClientServiceException, IOException;
 
@@ -65,6 +68,8 @@ void initialize(String server, int port, String accessKey, String secretKey,
    * @param bucket Bucket name
    * @param object Object path
    * @return S3ClientStat object
+   * @throws S3ClientServiceException if an S3 error occurred
+   * @throws IOException if an S3 error occurred
    */
   S3ClientStat statObject(String bucket, String object) throws S3ClientServiceException, IOException;
 
@@ -74,6 +79,8 @@ void initialize(String server, int port, String accessKey, String secretKey,
    * @param object Object path
    * @param offset Start reading at this offset
    * @return InputStream to the object
+   * @throws S3ClientServiceException if an S3 error occurred
+   * @throws IOException if an S3 error occurred
    */
   InputStream getObject(String bucket, String object, long offset) throws S3ClientServiceException, IOException;
 
@@ -82,6 +89,8 @@ void initialize(String server, int port, String accessKey, String secretKey,
    * @param bucket Bucket name
    * @param object Object path
    * @param filename Destination file
+   * @throws S3ClientServiceException if an S3 error occurred
+   * @throws IOException if an S3 error occurred
    */
   void getObject(String bucket, String object, String filename) throws S3ClientServiceException, IOException;