diff --git a/MergerLogic/Clients/S3Client.cs b/MergerLogic/Clients/S3Client.cs index f6992de..9b7c930 100644 --- a/MergerLogic/Clients/S3Client.cs +++ b/MergerLogic/Clients/S3Client.cs @@ -84,17 +84,18 @@ private bool IsKeyError(Exception e) { string methodName = MethodBase.GetCurrentMethod().Name; this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}"); - string keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, TileFormat.Jpeg, true); - byte[]? imageBytes = this.GetImageBytes(keyPrefix); + // Resolve key+extension with one LIST instead of speculatively downloading Jpeg-then-Png (up to two GETs per tile). + string? key = this.GetTileKey(z, x, y); + if (key == null) + { + return null; + } + + byte[]? imageBytes = this.GetImageBytes(key); if (imageBytes == null) { - keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, TileFormat.Png, true); - imageBytes = this.GetImageBytes(keyPrefix); - if (imageBytes == null) - { - return null; - } + return null; } this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}"); diff --git a/MergerLogicUnitTests/Utils/S3UtilsTest.cs b/MergerLogicUnitTests/Utils/S3UtilsTest.cs index 757cd2c..18decb2 100644 --- a/MergerLogicUnitTests/Utils/S3UtilsTest.cs +++ b/MergerLogicUnitTests/Utils/S3UtilsTest.cs @@ -177,81 +177,45 @@ public void GetTile(bool exist, GetTileParamType paramType, TileFormat tileForma using (var dataStream = new MemoryStream(data)) { - if (paramType != GetTileParamType.String) - { - this._pathUtilsMock - .InSequence(seq) - .Setup(utils => utils.GetTilePath("test", 0, 0, 0, TileFormat.Jpeg, true)) - .Returns("key"); - } - - if (exist) - { - this._amazonS3ClientMock - .Setup(s3 => s3.GetObjectAsync(It.Is(req => - req.BucketName == "bucket" && req.Key == "key"), It.IsAny())) - .ReturnsAsync(new GetObjectResponse() { ResponseStream = dataStream }); - this._imageFormatterMock.Setup(formatter => formatter.GetTileFormat(It.IsAny())) - .Returns(tileFormat); - - if (paramType == GetTileParamType.String) - { - this._s3ClientMock.Setup(s3 => s3.GetTile(It.IsAny())).Returns(new Tile(cords, data)); - } - else - { - this._s3ClientMock.Setup(s3 => s3.GetTile(It.IsAny(), It.IsAny(), - It.IsAny())).Returns(new Tile(cords, data)); - } - } - else + if (paramType == GetTileParamType.String) { this._amazonS3ClientMock .InSequence(seq) .Setup(s3 => s3.GetObjectAsync(It.Is(req => req.BucketName == "bucket" && req.Key == "key"), It.IsAny())) - .ThrowsAsync(new AmazonS3Exception("", Amazon.Runtime.ErrorType.Unknown, "NoSuchKey", "", System.Net.HttpStatusCode.NoContent)); - } - - if (paramType == GetTileParamType.String) - { + .ReturnsAsync(new GetObjectResponse() { ResponseStream = dataStream }); this._pathUtilsMock .InSequence(seq) .Setup(utils => utils.FromPath("key", true)) .Returns(cords); } - - // Repeat - if (paramType != GetTileParamType.String) + else { this._pathUtilsMock .InSequence(seq) - .Setup(utils => utils.GetTilePath("test", 0, 0, 0, TileFormat.Png, true)) - .Returns("key"); - } - - if (exist) - { - this._amazonS3ClientMock - .Setup(s3 => s3.GetObjectAsync(It.Is(req => - req.BucketName == "bucket" && req.Key == "key"), It.IsAny())) - .ReturnsAsync(new GetObjectResponse() { ResponseStream = dataStream }); - this._imageFormatterMock.Setup(formatter => formatter.GetTileFormat(It.IsAny())) - .Returns(tileFormat); + .Setup(utils => utils.GetTilePathWithoutExtension("test", 0, 0, 0, true)) + .Returns("keyPrefix"); - if (paramType != GetTileParamType.String) + var listResponse = new ListObjectsV2Response(); + if (exist) { - this._s3ClientMock.Setup(s3 => s3.GetTile(It.IsAny(), It.IsAny(), - It.IsAny())).Returns(new Tile(cords, data)); + listResponse.S3Objects.Add(new S3Object() { Key = "key" }); } - } - else - { this._amazonS3ClientMock .InSequence(seq) - .Setup(s3 => s3.GetObjectAsync(It.Is(req => - req.BucketName == "bucket" && req.Key == "key"), It.IsAny())) - .ThrowsAsync(new AmazonS3Exception("", Amazon.Runtime.ErrorType.Unknown, "NoSuchKey", "", System.Net.HttpStatusCode.NoContent)); + .Setup(s3 => s3.ListObjectsV2Async(It.Is(req => + req.BucketName == "bucket" && req.Prefix == "keyPrefix" && req.MaxKeys == 1), + It.IsAny())) + .ReturnsAsync(listResponse); + + if (exist) + { + this._amazonS3ClientMock + .InSequence(seq) + .Setup(s3 => s3.GetObjectAsync(It.Is(req => + req.BucketName == "bucket" && req.Key == "key"), It.IsAny())) + .ReturnsAsync(new GetObjectResponse() { ResponseStream = dataStream }); + } } var s3Utils = new S3Client(this._amazonS3ClientMock.Object, this._pathUtilsMock.Object, @@ -271,25 +235,18 @@ public void GetTile(bool exist, GetTileParamType paramType, TileFormat tileForma break; } - if (!exist) + if (paramType == GetTileParamType.String && !exist) + { + Assert.IsNull(tile); + } + else if (!exist) { Assert.IsNull(tile); + this._amazonS3ClientMock.Verify(s3 => s3.GetObjectAsync(It.IsAny(), It.IsAny()), Times.Never); } else { - if (paramType == GetTileParamType.String) - { - this._amazonS3ClientMock.Verify(s3 => s3.GetObjectAsync(It.IsAny(), It.IsAny()), Times.Once); - } - else if (!exist) - { - this._amazonS3ClientMock.Verify(s3 => s3.GetObjectAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); - } - else - { - this._amazonS3ClientMock.Verify(s3 => s3.GetObjectAsync(It.IsAny(), It.IsAny()), Times.Once()); - } - + this._amazonS3ClientMock.Verify(s3 => s3.GetObjectAsync(It.IsAny(), It.IsAny()), Times.Once); Assert.AreEqual(cords.Z, tile.Z); Assert.AreEqual(cords.X, tile.X); Assert.AreEqual(cords.Y, tile.Y); @@ -297,20 +254,6 @@ public void GetTile(bool exist, GetTileParamType paramType, TileFormat tileForma } } - if (paramType != GetTileParamType.String) - { - if (!exist) - { - this._pathUtilsMock.Verify(utils => utils.GetTilePath(It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); - } - else - { - this._pathUtilsMock.Verify(utils => utils.GetTilePath(It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny(), It.IsAny()), Times.Once()); - } - } - this.VerifyAll(); }