Add std.native & move std.xz/gzip to this (#241)

- Add support std.native
- Move our non-standard std.xz/std.gzip to this.
- Add gzip support to scala native.

Author: stephenamar-db

sjsonnet/src-native/sjsonnet/Platform.scala

@@ -1,12 +1,26 @@
 package sjsonnet
-import java.io.File
+
+import java.io.{ByteArrayOutputStream, File}
+import java.util.Base64
+import java.util.zip.GZIPOutputStream
+
 object Platform {
-  def gzipBytes(s: Array[Byte]): String = {
-    throw new Exception("GZip not implemented in Scala Native")
+  def gzipBytes(b: Array[Byte]): String = {
+    val outputStream: ByteArrayOutputStream = new ByteArrayOutputStream(b.length)
+    val gzip: GZIPOutputStream = new GZIPOutputStream(outputStream)
+    try {
+      gzip.write(b)
+    } finally {
+      gzip.close()
+      outputStream.close()
+    }
+    Base64.getEncoder.encodeToString(outputStream.toByteArray)
   }
+
   def gzipString(s: String): String = {
-    throw new Exception("GZip not implemented in Scala Native")
+    gzipBytes(s.getBytes())
   }
+
   def xzBytes(s: Array[Byte], compressionLevel: Option[Int]): String = {
     throw new Exception("XZ not implemented in Scala Native")
   }

sjsonnet/src/sjsonnet/ReadWriter.scala

@@ -44,4 +44,8 @@ object ReadWriter{
     def apply(t: Val) = t.asFunc
     def write(pos: Position, t: Val.Func) = t
   }
+  implicit object BuiltinRead  extends ReadWriter[Val.Builtin] {
+    def apply(t: Val) = t.asInstanceOf[Val.Builtin]
+    def write(pos: Position, t: Val.Builtin) = t
+  }
 }

sjsonnet/src/sjsonnet/Std.scala

@@ -16,11 +16,39 @@ import scala.util.matching.Regex
   * in Scala code. Uses `builtin` and other helpers to handle the common wrapper
   * logic automatically
   */
-class Std {
+class Std(private val additionalNativeFunctions: Map[String, Val.Builtin] = Map.empty) {
   private val dummyPos: Position = new Position(null, 0)
   private val emptyLazyArray = new Array[Lazy](0)
   private val leadingWhiteSpacePattern = Pattern.compile("^[ \t\n\f\r\u0085\u00A0']+")
   private val trailingWhiteSpacePattern = Pattern.compile("[ \t\n\f\r\u0085\u00A0']+$")
+  private val oldNativeFunctions = Map(
+    builtin("gzip", "v"){ (_, _, v: Val) =>
+      v match{
+        case Val.Str(_, value) => Platform.gzipString(value)
+        case arr: Val.Arr => Platform.gzipBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray)
+        case x => Error.fail("Cannot gzip encode " + x.prettyName)
+      }
+    },
+
+    builtinWithDefaults("xz", "v" -> null, "compressionLevel" -> Val.Null(dummyPos)){ (args, pos, ev) =>
+      val compressionLevel: Option[Int] = args(1) match {
+        case Val.Null(_) =>
+          // Use default compression level if the user didn't set one
+          None
+        case Val.Num(_, n) =>
+          Some(n.toInt)
+        case x =>
+          Error.fail("Cannot xz encode with compression level " + x.prettyName)
+      }
+      args(0) match {
+        case Val.Str(_, value) => Platform.xzString(value, compressionLevel)
+        case arr: Val.Arr => Platform.xzBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray, compressionLevel)
+        case x => Error.fail("Cannot xz encode " + x.prettyName)
+      }
+    },
+  )
+  require(oldNativeFunctions.forall(k => !additionalNativeFunctions.contains(k._1)), "Conflicting native functions")
+  private val nativeFunctions = oldNativeFunctions ++ additionalNativeFunctions
 
   private object AssertEqual extends Val.Builtin2("assertEqual", "a", "b") {
     def evalRhs(v1: Val, v2: Val, ev: EvalScope, pos: Position): Val = {
@@ -1285,31 +1313,6 @@ class Std {
       new Val.Arr(pos, Base64.getDecoder().decode(s).map(i => Val.Num(pos, i)))
     },
 
-    builtin("gzip", "v"){ (pos, ev, v: Val) =>
-      v match{
-        case Val.Str(_, value) => Platform.gzipString(value)
-        case arr: Val.Arr => Platform.gzipBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray)
-        case x => Error.fail("Cannot gzip encode " + x.prettyName)
-      }
-    },
-
-    builtinWithDefaults("xz", "v" -> null, "compressionLevel" -> Val.Null(dummyPos)){ (args, pos, ev) =>
-      val compressionLevel: Option[Int] = args(1) match {
-        case Val.Null(_) =>
-          // Use default compression level if the user didn't set one
-          None
-        case Val.Num(_, n) =>
-          Some(n.toInt)
-        case x =>
-          Error.fail("Cannot xz encode with compression level " + x.prettyName)
-      }
-      args(0) match {
-        case Val.Str(_, value) => Platform.xzString(value, compressionLevel)
-        case arr: Val.Arr => Platform.xzBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray, compressionLevel)
-        case x => Error.fail("Cannot xz encode " + x.prettyName)
-      }
-    },
-
     builtin(EncodeUTF8),
     builtin(DecodeUTF8),
 
@@ -1511,8 +1514,15 @@ class Std {
     builtin(MaxArray),
     builtin("primitiveEquals", "x", "y") { (_, ev, x: Val, y: Val) =>
       x.isInstanceOf[y.type] && ev.compare(x, y) == 0
-    }
-  )
+    },
+    builtin("native", "name") { (pos, ev, name: String) =>
+      if (nativeFunctions.contains(name)) {
+        nativeFunctions(name)
+      } else {
+        Error.fail("Native function " + name + " not found", pos)(ev)
+      }
+    },
+  ) ++ oldNativeFunctions
 
   private def toSetArrOrString(args: Array[Val], idx: Int, pos: Position, ev: EvalScope) = {
     args(idx) match {

sjsonnet/test/src-jvm-native/ErrorTests.scala

@@ -259,9 +259,8 @@ object ErrorTests extends TestSuite{
       )
     }
     test("native_not_found") - check(
-      """sjsonnet.Error: Field does not exist: native
-        |    at [Select native].(sjsonnet/test/resources/test_suite/error.native_not_found.jsonnet:17:4)
-        |    at [Apply1].(sjsonnet/test/resources/test_suite/error.native_not_found.jsonnet:17:11)
+      """sjsonnet.Error: Native function non_existent_native not found
+        |    at [std.native].(sjsonnet/test/resources/test_suite/error.native_not_found.jsonnet:17:11)
         |""".stripMargin
     )
     test("obj_assert") - {

sjsonnet/test/src-jvm/sjsonnet/Example.java

@@ -1,5 +1,7 @@
 package sjsonnet;
 
+import scala.collection.immutable.Map$;
+
 public class Example {
     public void example(){
         sjsonnet.SjsonnetMain.main0(
@@ -11,7 +13,7 @@ public void example(){
             os.package$.MODULE$.pwd(),
             scala.None$.empty(),
             scala.None$.empty(),
-            new sjsonnet.Std().Std()
+            new sjsonnet.Std(Map$.MODULE$.empty()).Std()
         );
     }
 }

sjsonnet/test/src-jvm/sjsonnet/StdGzipTests.scala

@@ -16,6 +16,16 @@ object StdGzipTests extends TestSuite {
         case s if s >= 16 => "H4sIAAAAAAAA/8vIBACsKpPYAgAAAA=="
         case _ => "H4sIAAAAAAAAAMvIBACsKpPYAgAAAA=="
       })
+      eval("""std.native('gzip')([1, 2])""") ==> ujson.Str(Runtime.version().feature() match {
+        // https://bugs.openjdk.org/browse/JDK-8244706
+        case s if s >= 16 => "H4sIAAAAAAAA/2NkAgCSQsy2AgAAAA=="
+        case _ => "H4sIAAAAAAAAAGNkAgCSQsy2AgAAAA=="
+      })
+      eval("""std.native('gzip')("hi")""") ==> ujson.Str(Runtime.version().feature() match {
+        // https://bugs.openjdk.org/browse/JDK-8244706
+        case s if s >= 16 => "H4sIAAAAAAAA/8vIBACsKpPYAgAAAA=="
+        case _ => "H4sIAAAAAAAAAMvIBACsKpPYAgAAAA=="
+      })
     }
   }
 }

sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala

@@ -10,11 +10,20 @@ object StdXzTests extends TestSuite {
       eval("""std.xz("hi")""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARYAAAB0L+WjAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla")
       eval("""std.xz([1, 2], compressionLevel = 0)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhAQwAAACPmEGcAQABAQIAAADRC9qlUgJ94gABGgLcLqV+H7bzfQEAAAAABFla")
       eval("""std.xz("hi", compressionLevel = 1)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARAAAACocI6GAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla")
-      val ex = intercept[Exception] {
+      var ex = intercept[Exception] {
         // Compression level 10 is invalid
         eval("""std.xz("hi", 10)""")
       }
       assert(ex.getMessage.contains("Unsupported preset: 10"))
+      eval("""std.native('xz')([1, 2])""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARYAAAB0L+WjAQABAQIAAADRC9qlUgJ94gABGgLcLqV+H7bzfQEAAAAABFla")
+      eval("""std.native('xz')("hi")""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARYAAAB0L+WjAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla")
+      eval("""std.native('xz')([1, 2], compressionLevel = 0)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhAQwAAACPmEGcAQABAQIAAADRC9qlUgJ94gABGgLcLqV+H7bzfQEAAAAABFla")
+      eval("""std.native('xz')("hi", compressionLevel = 1)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARAAAACocI6GAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla")
+      ex = intercept[Exception] {
+        // Compression level 10 is invalid
+        eval("""std.native('xz')("hi", 10)""")
+      }
+      assert(ex.getMessage.contains("Unsupported preset: 10"))
     }
   }
 }

sjsonnet/test/src-native/sjsonnet/StdGzipTests.scala

@@ -0,0 +1,16 @@
+package sjsonnet
+
+import sjsonnet.TestUtils.eval
+import utest._
+
+object StdGzipTests extends TestSuite {
+  val tests = Tests {
+    test("gzip"){
+      eval("""std.gzip([1, 2])""") ==> ujson.Str("H4sIAAAAAAAAAGNkAgCSQsy2AgAAAA==")
+      eval("""std.gzip("hi")""") ==> ujson.Str("H4sIAAAAAAAAAMvIBACsKpPYAgAAAA==")
+      eval("""std.native('gzip')([1, 2])""") ==> ujson.Str("H4sIAAAAAAAAAGNkAgCSQsy2AgAAAA==")
+      eval("""std.native('gzip')("hi")""") ==> ujson.Str("H4sIAAAAAAAAAMvIBACsKpPYAgAAAA==")
+    }
+  }
+}
+