Merge branch 'master' of github.com:findify/s3mock

Author: shuttie

CHANGELOG.md

@@ -1,3 +1,13 @@
+0.2.4 (not yet released)
+=======
+* pom -> jar dependency type doc fix
+* support alpakka multipart uploads
+* support alpakka listObjects ([#66](https://github.com/findify/s3mock/issues/66))
+* fix bug with etag on FileProvider being alsays "0" ([#70](https://github.com/findify/s3mock/issues/70))
+* fix last-modified header always being equal to "1970-01-01 00:00:00"([65](https://github.com/findify/s3mock/issues/70))
+* wrong content-type for listObjects ([#60](https://github.com/findify/s3mock/issues/60))
+* deleteObjects broken on aws s3 sdk 2.0 ([#71](https://github.com/findify/s3mock/issues/60))
+
 0.2.3
 =======
 * windows compatibility in FileProvider ([#28](https://github.com/findify/s3mock/issues/28))

README.md

@@ -37,7 +37,6 @@ On maven, update your `pom.xml` in the following way:
         <groupId>io.findify</groupId>
         <artifactId>s3mock_2.12</artifactId>
         <version>0.2.3</version>
-        <type>pom</type>
         <scope>test</scope>
     </dependency>
 ```

build.sbt

@@ -4,11 +4,11 @@ version := "0.2.3"
 
 organization := "io.findify"
 
-scalaVersion := "2.12.3"
+scalaVersion := "2.12.4"
 
-crossScalaVersions := Seq("2.11.11", "2.12.2")
+crossScalaVersions := Seq("2.11.11", "2.12.4")
 
-val akkaVersion = "2.5.4"
+val akkaVersion = "2.5.6"
 
 licenses := Seq("MIT" -> url("https://opensource.org/licenses/MIT"))
 
@@ -21,11 +21,11 @@ libraryDependencies ++= Seq(
   "org.scala-lang.modules" %% "scala-xml" % "1.0.6",
   "com.github.pathikrit" %% "better-files" % "2.17.1",
   "com.typesafe.scala-logging" %% "scala-logging" % "3.7.2",
-  "com.amazonaws" % "aws-java-sdk-s3" % "1.11.192",
+  "com.amazonaws" % "aws-java-sdk-s3" % "1.11.224",
   "org.scalatest" %% "scalatest" % "3.0.4" % "test",
   "ch.qos.logback" % "logback-classic" % "1.2.3" % "test",
   "org.iq80.leveldb" % "leveldb" % "0.9",
-  "com.lightbend.akka" %% "akka-stream-alpakka-s3" % "0.11" % "test"
+  "com.lightbend.akka" %% "akka-stream-alpakka-s3" % "0.14" % "test"
 )
 
 parallelExecution in Test := false
@@ -51,4 +51,13 @@ pomExtra := (
         <name>Roman Grebennikov</name>
         <url>http://www.dfdx.me</url>
       </developer>
-    </developers>)
+    </developers>)
+
+/*enablePlugins(JavaAppPackaging)
+
+maintainer in Docker := "S3mock"
+packageSummary in Docker := "S3Mock"
+packageDescription := "Mock Service For S3"
+dockerUpdateLatest := true
+dockerExposedPorts := Seq(8001)
+*/

project/plugins.sbt

@@ -4,4 +4,5 @@ addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.3.0")
 addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2")
 addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
 addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.1")
-addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.4")
+addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.4")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.2")

src/main/scala/io/findify/s3mock/S3Mock.scala

@@ -37,7 +37,8 @@ class S3Mock(port:Int, provider:Provider)(implicit system:ActorSystem = ActorSys
           concat(
             ListBucket().route(bucket),
             CreateBucket().route(bucket),
-            DeleteBucket().route(bucket)
+            DeleteBucket().route(bucket),
+            DeleteObjects().route(bucket)
           )
         } ~ parameterMap { params =>
           path(RemainingPath) { key =>
@@ -59,7 +60,7 @@ class S3Mock(port:Int, provider:Provider)(implicit system:ActorSystem = ActorSys
         }
       }
 
-    bind = Await.result(http.bindAndHandle(route, "localhost", port), Duration.Inf)
+    bind = Await.result(http.bindAndHandle(route, "0.0.0.0", port), Duration.Inf)
     bind
   }
 

src/main/scala/io/findify/s3mock/provider/FileProvider.scala

@@ -1,6 +1,6 @@
 package io.findify.s3mock.provider
 import java.util.UUID
-import java.io.{File => JFile}
+import java.io.{FileInputStream, File => JFile}
 
 import akka.http.scaladsl.model.DateTime
 import better.files.File
@@ -11,6 +11,7 @@ import io.findify.s3mock.error.{NoSuchBucketException, NoSuchKeyException}
 import io.findify.s3mock.provider.metadata.{MapMetadataStore, MetadataStore}
 import io.findify.s3mock.request.{CompleteMultipartUpload, CreateBucketConfiguration}
 import io.findify.s3mock.response._
+import org.apache.commons.codec.digest.DigestUtils
 
 import scala.util.Random
 
@@ -46,7 +47,11 @@ class FileProvider(dir:String) extends Provider with LazyLogging {
         val fString = fromOs(f.toString).drop(bucketFileString.length).dropWhile(_ == '/')
         fString.startsWith(prefixNoLeadingSlash) && !f.isDirectory
       })
-    val files = bucketFiles.map(f => {Content(fromOs(f.toString).drop(bucketFileString.length+1).dropWhile(_ == '/'), DateTime(f.lastModifiedTime.toEpochMilli), "0", f.size, "STANDARD")}).toList
+    val files = bucketFiles.map(f => {
+      val stream = new FileInputStream(f.toJava)
+      val md5 = DigestUtils.md5Hex(stream)
+      Content(fromOs(f.toString).drop(bucketFileString.length+1).dropWhile(_ == '/'), DateTime(f.lastModifiedTime.toEpochMilli), md5, f.size, "STANDARD")
+    }).toList
     logger.debug(s"listing bucket contents: ${files.map(_.key)}")
     val commonPrefixes = delimiter match {
       case Some(del) => files.flatMap(f => commonPrefix(f.key, prefixNoLeadingSlash, del)).distinct.sorted
@@ -71,6 +76,7 @@ class FileProvider(dir:String) extends Provider with LazyLogging {
     file.createIfNotExists(createParents = true)
     logger.debug(s"writing file for s3://$bucket/$key to $dir/$bucket/$key, bytes = ${data.length}")
     file.writeByteArray(data)(OpenOptions.default)
+    objectMetadata.setLastModified(org.joda.time.DateTime.now().toDate)
     metadataStore.put(bucket, key, objectMetadata)
   }
   override def getObject(bucket:String, key:String): GetObjectData = {

src/main/scala/io/findify/s3mock/provider/InMemoryProvider.scala

@@ -1,6 +1,7 @@
 package io.findify.s3mock.provider
 
-import java.util.UUID
+import java.time.Instant
+import java.util.{Date, UUID}
 
 import akka.http.scaladsl.model.DateTime
 import com.amazonaws.services.s3.model.ObjectMetadata
@@ -75,6 +76,7 @@ class InMemoryProvider extends Provider with LazyLogging {
       case Some(bucketContent) =>
         logger.debug(s"putting object for s3://$bucket/$key, bytes = ${data.length}")
         bucketContent.keysInBucket.put(key, KeyContents(DateTime.now, data))
+        objectMetadata.setLastModified(org.joda.time.DateTime.now().toDate)
         metadataStore.put(bucket, key, objectMetadata)
       case None => throw NoSuchBucketException(bucket)
     }

src/main/scala/io/findify/s3mock/route/GetObject.scala

@@ -2,6 +2,7 @@ package io.findify.s3mock.route
 
 import java.io.StringWriter
 import java.net.URLDecoder
+import java.util.Date
 
 import akka.http.scaladsl.model.HttpEntity.Strict
 import akka.http.scaladsl.model._
@@ -118,10 +119,15 @@ case class GetObject(implicit provider: Provider) extends LazyLogging {
   }
 
   val headerBlacklist = Set("content-type", "connection")
-  protected def metadataToHeaderList(metadata: ObjectMetadata): List[RawHeader] = {
+  protected def metadataToHeaderList(metadata: ObjectMetadata): List[HttpHeader] = {
     val headers = Option(metadata.getRawMetadata)
       .map(_.asScala.toMap)
-      .map(_.map { case (key, value) => RawHeader(key, value.toString)}.toList)
+      .map(_.map {
+        case (_, date: Date) =>
+          `Last-Modified`(DateTime(new org.joda.time.DateTime(date).getMillis))
+        case (key, value) =>
+          RawHeader(key, value.toString)
+      }.toList)
       .toList.flatten
       .filterNot(header => headerBlacklist.contains(header.lowercaseName))
 

src/main/scala/io/findify/s3mock/route/ListBucket.scala

@@ -1,6 +1,6 @@
 package io.findify.s3mock.route
 
-import akka.http.scaladsl.model.{HttpResponse, StatusCodes}
+import akka.http.scaladsl.model._
 import akka.http.scaladsl.server.Directives._
 import com.typesafe.scalalogging.LazyLogging
 import io.findify.s3mock.error.{InternalErrorException, NoSuchBucketException}
@@ -17,16 +17,19 @@ case class ListBucket(implicit provider:Provider) extends LazyLogging {
       complete {
         logger.info(s"listing bucket $bucket with prefix=$prefix, delimiter=$delimiter")
         Try(provider.listBucket(bucket, prefix, delimiter, maxkeys.map(_.toInt))) match {
-          case Success(l) => HttpResponse(StatusCodes.OK, entity = l.toXML.toString)
+          case Success(l) => HttpResponse(
+            StatusCodes.OK,
+            entity = HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), l.toXML.toString)
+          )
           case Failure(e: NoSuchBucketException) =>
             HttpResponse(
               StatusCodes.NotFound,
-              entity = e.toXML.toString()
+              entity = HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), e.toXML.toString)
             )
           case Failure(t) =>
             HttpResponse(
               StatusCodes.InternalServerError,
-              entity = InternalErrorException(t).toXML.toString()
+              entity = HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), InternalErrorException(t).toXML.toString)
             )
         }
       }

src/main/scala/io/findify/s3mock/route/ListBuckets.scala

@@ -1,6 +1,6 @@
 package io.findify.s3mock.route
 
-import akka.http.scaladsl.model.{HttpResponse, StatusCodes}
+import akka.http.scaladsl.model._
 import akka.http.scaladsl.server.Directives._
 import com.typesafe.scalalogging.LazyLogging
 import io.findify.s3mock.provider.Provider
@@ -12,7 +12,10 @@ case class ListBuckets(implicit provider:Provider) extends LazyLogging {
   def route() = get {
     complete {
       logger.debug("listing all buckets")
-      HttpResponse(StatusCodes.OK, entity = provider.listBuckets.toXML.toString)
+      HttpResponse(
+        StatusCodes.OK,
+        entity = HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), provider.listBuckets.toXML.toString)
+      )
     }
   }
 }