Merge branch 'dev' into glfw3-support-for-osx

.gitattributes

@@ -1,34 +1,39 @@
 *.dae binary
 
 # Defold Protocol Buffer Text Files
-*.animationset linguist-language=text-proto
-*.atlas linguist-language=text-proto
-*.camera linguist-language=text-proto
-*.collection linguist-language=text-proto
-*.collectionfactory linguist-language=text-proto
-*.collectionproxy linguist-language=text-proto
-*.collisionobject linguist-language=text-proto
-*.cubemap linguist-language=text-proto
-*.display_profiles linguist-language=text-proto
-*.factory linguist-language=text-proto
-*.font linguist-language=text-proto
-*.gamepads linguist-language=text-proto
-*.go linguist-language=text-proto
-*.gui linguist-language=text-proto
-*.input_binding linguist-language=text-proto
-*.label linguist-language=text-proto
-*.material linguist-language=text-proto
-*.mesh linguist-language=text-proto
-*.model linguist-language=text-proto
-*.particlefx linguist-language=text-proto
-*.render linguist-language=text-proto
-*.sound linguist-language=text-proto
-*.sprite linguist-language=text-proto
-*.spinemodel linguist-language=text-proto
-*.spinescene linguist-language=text-proto
-*.texture_profiles linguist-language=text-proto
-*.tilemap linguist-language=text-proto
-*.tilesource linguist-language=text-proto
+*.animationset linguist-language=Protocol-Buffer-Text-Format
+*.atlas linguist-language=Protocol-Buffer-Text-Format
+*.camera linguist-language=Protocol-Buffer-Text-Format
+*.collection linguist-language=Protocol-Buffer-Text-Format
+*.collectionfactory linguist-language=Protocol-Buffer-Text-Format
+*.collectionproxy linguist-language=Protocol-Buffer-Text-Format
+*.collisionobject linguist-language=Protocol-Buffer-Text-Format
+*.cubemap linguist-language=Protocol-Buffer-Text-Format
+*.display_profiles linguist-language=Protocol-Buffer-Text-Format
+*.factory linguist-language=Protocol-Buffer-Text-Format
+*.font linguist-language=Protocol-Buffer-Text-Format
+*.gamepads linguist-language=Protocol-Buffer-Text-Format
+*.go linguist-language=Protocol-Buffer-Text-Format
+*.gui linguist-language=Protocol-Buffer-Text-Format
+*.input_binding linguist-language=Protocol-Buffer-Text-Format
+*.label linguist-language=Protocol-Buffer-Text-Format
+*.material linguist-language=Protocol-Buffer-Text-Format
+*.mesh linguist-language=Protocol-Buffer-Text-Format
+*.model linguist-language=Protocol-Buffer-Text-Format
+*.particlefx linguist-language=Protocol-Buffer-Text-Format
+*.render linguist-language=Protocol-Buffer-Text-Format
+*.sound linguist-language=Protocol-Buffer-Text-Format
+*.sprite linguist-language=Protocol-Buffer-Text-Format
+*.spinemodel linguist-language=Protocol-Buffer-Text-Format
+*.spinescene linguist-language=Protocol-Buffer-Text-Format
+*.texture_profiles linguist-language=Protocol-Buffer-Text-Format
+*.tilemap linguist-language=Protocol-Buffer-Text-Format
+*.tilesource linguist-language=Protocol-Buffer-Text-Format
+
+# Defold YAML Files
+*.appmanifest linguist-language=YAML
+*.manifest linguist-language=YAML
+*.script_api linguist-language=YAML
 
 # Defold JSON Files
 *.buffer linguist-language=JSON

RELEASE.md

@@ -28,11 +28,13 @@ The alpha channel is automatically released for every successful push to dev.
   $ git merge dev
   $ git push
 
- Beta beta channel is automatically released for every successful push to beta.
+ Beta channel is automatically released for every successful push to beta.
 
-1. Collect release notes using `python scripts/releasenotes_git.py` and post on [forum.defold.com](https://forum.defold.com/c/releasenotes)
+1. Collect release notes using `python scripts/releasenotes_github_projectv2.py` and post on [forum.defold.com](https://forum.defold.com/c/releasenotes)
 and add the "BETA" tag to the headline
 
+* Note: The release notes script requires a github access token to work correctly (https://github.com/settings/tokens). If the token is incorrect, you will likely get an error saying 'Unable to find GitHub project for version x.x.x'. Create a "classic" github token and add permissions to read projects. If the script still fails, you might need more permissions.
+
 ### Update private repos
 
 1. Also update the beta for the console platforms

build_tools/s3.py

@@ -189,10 +189,11 @@ def move_release(archive_path, sha1, channel):
  continue
 
  # copy the file to the new location
- print("Copy object: %s -> %s" % (obj.key, new_key))
- print("Create redirection %s to %s\n" % (obj.key, new_redirect))
  new_object = bucket.Object(new_key)
  new_redirect = "http://%s/%s" % (bucket_name, new_key)
+
+ print("Copy object: %s -> %s" % (obj.key, new_key))
+ print("Create redirection %s to %s\n" % (obj.key, new_redirect))
  new_object.copy_from(
  CopySource={'Bucket': bucket_name, 'Key': obj.key}
  )

editor/project.clj

@@ -139,7 +139,8 @@
  "prerelease" ["do" "clean," "protobuf," "sass" "once," "javac," "with-profile" "dev,sidecar,release" "run" "-m" "aot"]}
 
  ;; used by `pack` task
- :packing {:pack-path "resources/_unpack"}
+ :packing {:pack-path "resources/_unpack"
+ :lua-language-server-version "v1.7793"}
 
  :codox {:sources ["src/clj"]
  :output-dir "target/doc/api"

editor/src/clj/editor/editor_extensions.clj

@@ -791,14 +791,21 @@
  :when dynamic-handler]
  dynamic-handler))))
 
+(defn- built-in-language-servers []
+ (let [lua-lsp-root (str (system/defold-unpack-path) "/" (.getPair (Platform/getHostPlatform)) "/bin/lsp/lua")]
+ #{{:languages #{"lua"}
+ :watched-files [{:pattern "**/.luacheckrc"}]
+ :launcher {:command [(str lua-lsp-root "/bin/lua-language-server" (when (util/is-win32?) ".exe"))
+ (str "--configpath=" lua-lsp-root "/config.json")]}}}))
+
 (defn- reload-language-servers! [project]
  (let [lsp (lsp/get-node-lsp project)
  state (ext-state project)
  ui (:ui state)]
  (lsp/set-servers!
  lsp
  (into
- #{}
+ (built-in-language-servers)
  (comp
  (mapcat
  (fn [[path language-servers]]

editor/src/clj/util/net.clj

@@ -14,10 +14,7 @@
 
 (ns util.net
  (:require [clojure.java.io :as io])
- (:import
- (java.net URI URL HttpURLConnection)))
-
-(import '(java.net HttpURLConnection))
+ (:import [java.net HttpURLConnection]))
 
 (set! *warn-on-reflection* true)
 
@@ -34,31 +31,30 @@
  connect-timeout default-connect-timeout
  chunk-size 0
  cancelled-derefable default-cancelled-derefable
- progress-callback default-progress-callback}}]
+ progress-callback default-progress-callback}
+ :as args}]
  (let [^HttpURLConnection conn (doto (.openConnection (io/as-url url))
- (.setRequestProperty "Connection" "close")
- (.setConnectTimeout connect-timeout)
- (.setReadTimeout read-timeout))
+  (.setRequestProperty "Connection" "close")
+  (.setConnectTimeout connect-timeout)
+  (.setReadTimeout read-timeout))
  length (.getContentLengthLong conn)
  response-code (.getResponseCode conn)]
-
  (case response-code
- 301 (let [location (.getHeaderField conn "Location")]
- (println (format "moved permanently '%s' -> '%s'" url location))
- (download! location out))
- 200 (when-not @cancelled-derefable
- (with-open [input (.getInputStream conn)
- output (io/output-stream out)]
- (if (pos? chunk-size)
- (let [buf (byte-array chunk-size)]
- (loop [count (.read input buf)
- previous 0]
- (when (and (<= 0 count) (not @cancelled-derefable))
- (.write output buf 0 count)
- (progress-callback (+ previous count) length)
- (recur (.read input buf) (+ previous count)))))
- (do
- ;; io/copy will use an internal buffer of 1024 bytes for transfer
- (io/copy input output)
- (when (not @cancelled-derefable)
- (progress-callback length length)))))))))
+ 301 (let [location (.getHeaderField conn "Location")]
+ (apply download! location out (mapcat identity args)))
+ 200 (when-not @cancelled-derefable
+ (with-open [input (.getInputStream conn)
+ output (io/output-stream out)]
+ (if (pos? chunk-size)
+ (let [buf (byte-array chunk-size)]
+ (loop [count (.read input buf)
+ previous 0]
+ (when (and (<= 0 count) (not @cancelled-derefable))
+ (.write output buf 0 count)
+ (progress-callback (+ previous count) length)
+ (recur (.read input buf) (+ previous count)))))
+ (do
+ ;; io/copy will use an internal buffer of 1024 bytes for transfer
+ (io/copy input output)
+ (when (not @cancelled-derefable)
+ (progress-callback length length)))))))))

editor/src/java/com/defold/libs/ResourceUnpacker.java

@@ -166,7 +166,6 @@ private static void unpackResourceFile(String resourceFileName, Path target, boo
  FileUtils.deleteQuietly(outputFile);
  }
 
- logger.debug("unpacking file '{}' to '{}'", resourceFileName, outputPath);
  File outputDirectory = outputFile.getParentFile();
 
  if (outputDirectory != null) {
@@ -210,7 +209,6 @@ private static void unpackResourceDir(String resourceDir, Path target) throws IO
  if (destFile.exists() && destFile.isDirectory()) {
  FileUtils.deleteQuietly(destFile);
  }
- logger.debug("unpacking '{}' to '{}'", source, dest);
  try {
  Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
  }

editor/tasks/leiningen/pack.clj

@@ -13,13 +13,16 @@
 ;; specific language governing permissions and limitations under the License.
 
 (ns leiningen.pack
- (:require
- [clojure.java.io :as io]
- [clojure.string :as str]
- [leiningen.util.http-cache :as http-cache])
- (:import
- (java.util.zip ZipFile)
- (org.apache.commons.io FileUtils)))
+ (:require [clojure.java.io :as io]
+ [clojure.string :as str]
+ [leiningen.util.http-cache :as http-cache])
+ (:import [java.io File]
+ [java.nio.file CopyOption Files FileSystems FileVisitor FileVisitResult LinkOption Path Paths]
+ [java.nio.file.attribute FileAttribute]
+ [java.util.zip ZipEntry ZipFile]
+ [org.apache.commons.io FileUtils]))
+
+(set! *warn-on-reflection* true)
 
 (defn dynamo-home [] (get (System/getenv) "DYNAMO_HOME"))
 
@@ -88,7 +91,7 @@
 
 (defn artifact-files
  []
- (let [subst (fn [s] (.replace s "${DYNAMO-HOME}" (dynamo-home)))]
+ (let [subst (fn [s] (str/replace s "${DYNAMO-HOME}" (dynamo-home)))]
  (into {} (for [[src dest] artifacts]
  [(io/file (subst src)) (io/file (subst dest))]))))
 
@@ -102,7 +105,7 @@
  "windows-x64" ["x86_64-win32"]})
 
 (defn jar-file
- [[artifact version & {:keys [classifier]} :as dependency]]
+ ^File [[artifact version & {:keys [classifier]} :as dependency]]
  (io/file (str (System/getProperty "user.home")
  "/.m2/repository/"
  (str/replace (namespace artifact) "." "/")
@@ -122,7 +125,7 @@
  (let [java-platform (str/replace-first classifier "natives-" "")
  natives-path (str "natives/" java-platform)]
  (with-open [zip-file (ZipFile. (jar-file dependency))]
- (doseq [entry (enumeration-seq (.entries zip-file))]
+ (doseq [^ZipEntry entry (enumeration-seq (.entries zip-file))]
  (when (.startsWith (.getName entry) natives-path)
  (let [libname (.getName (io/file (.getName entry)))]
  (doseq [target-platform (jogl-classifier->platforms java-platform)]
@@ -136,11 +139,55 @@
  (doseq [jogl-native-dep (filter jogl-native-dep? dependencies)]
  (extract-jogl-native-dep jogl-native-dep pack-path)))
 
+(defn pack-lua-language-server [pack-path lua-language-server-version]
+ (let [release-path (-> (format "https://github.com/defold/lua-language-server/releases/download/%s/release.zip"
+ lua-language-server-version)
+ http-cache/download
+ .toPath)
+ file-attributes (into-array FileAttribute [])
+ ^"[Ljava.nio.file.CopyOption;" copy-options (into-array CopyOption [])]
+ (with-open [fs (FileSystems/newFileSystem release-path)]
+ (doseq [platform (keys engine-artifacts)
+ :let [zip-file-name (str platform ".zip")
+ src-zip-path (.getPath fs "lsp-lua-language-server" (into-array String ["plugins" zip-file-name]))
+ dst-root-path (Paths/get pack-path (into-array String [platform "bin" "lsp" "lua"]))]]
+ ;; Copy config.json to the pack path
+ (let [source-path (.getPath fs "lsp-lua-language-server" (into-array String ["plugins" "share" "config.json"]))
+ target-path (.resolve dst-root-path "config.json")]
+ (Files/createDirectories (.getParent target-path) file-attributes)
+ (Files/copy source-path target-path copy-options))
+ ;; Copy contents of bin zips to the pack path
+ (with-open [fs (FileSystems/newFileSystem src-zip-path)]
+ (doseq [^Path root-path (.getRootDirectories fs)
+ :let [entry-path->dst-path (fn [^Path p]
+ (let [name-count (.getNameCount p)]
+ (when (< 2 name-count)
+ (.resolve dst-root-path
+ (-> root-path
+ (.relativize p)
+ ;; remove leading "bin/${platform}"
+ (.subpath 2 name-count)
+ str)))))]]
+ (Files/walkFileTree
+ root-path
+ (reify FileVisitor
+ (preVisitDirectory [_ path _]
+ (when-let [^Path target-path (entry-path->dst-path path)]
+ (when-not (Files/exists target-path (into-array LinkOption []))
+ (Files/createDirectories target-path file-attributes)))
+ FileVisitResult/CONTINUE)
+ (visitFile [_ path _]
+ (when-let [^Path target-path (entry-path->dst-path path)]
+ (Files/deleteIfExists target-path)
+ (Files/copy ^Path path target-path copy-options))
+ FileVisitResult/CONTINUE)
+ (postVisitDirectory [_ _ _] FileVisitResult/CONTINUE)))))))))
+
 (defn copy-artifacts
  [pack-path archive-domain git-sha]
  (let [files (merge (engine-artifact-files archive-domain git-sha)
  (artifact-files))]
- (doseq [[src dest] files]
+ (doseq [[^File src dest] files]
  (let [dest (io/file pack-path dest)]
  (println (format "copying '%s' to '%s'" (str src) (str dest)))
  (if-not (.exists src)
@@ -154,7 +201,8 @@
  [{:keys [dependencies packing] :as project} & [git-sha]]
  (let [sha (or git-sha (:engine project))
  archive-domain (get project :archive-domain)
- {:keys [pack-path]} packing]
+ {:keys [pack-path lua-language-server-version]} packing]
  (FileUtils/deleteQuietly (io/file pack-path))
  (copy-artifacts pack-path archive-domain sha)
- (pack-jogl-natives pack-path dependencies)))
+ (pack-jogl-natives pack-path dependencies)
+ (pack-lua-language-server pack-path lua-language-server-version)))

editor/tasks/leiningen/util/http_cache.clj

@@ -109,7 +109,8 @@
  (catch FileNotFoundException e
  {:code 404}))))
 
-(defn download [url]
+(defn download
+ ^File [url]
  (let [c (->cache "~/.dcache" (* 4 1000000000))
  hit (cache-get c url)
  headers (if hit

engine/gamesys/src/gamesys/components/comp_collection_proxy.cpp

@@ -799,16 +799,18 @@ namespace dmGameSystem
 
  /*# return an indexed table of all the resources of a collection proxy
  *
- * return an indexed table of resources for a collection proxy. Each
- * entry is a hexadecimal string that represents the data of the specific
- * resource. This representation corresponds with the filename for each
- * individual resource that is exported when you bundle an application with
- * LiveUpdate functionality.
+ * return an indexed table of resources for a collection proxy where the
+ * referenced collection has been excluded using LiveUpdate. Each entry is a
+ * hexadecimal string that represents the data of the specific resource.
+ * This representation corresponds with the filename for each individual
+ * resource that is exported when you bundle an application with LiveUpdate
+ * functionality.
  *
  * @namespace collectionproxy
  * @name collectionproxy.get_resources
  * @param collectionproxy [type:url] the collectionproxy to check for resources.
- * @return resources [type:table] the resources
+ * @return resources [type:table] the resources, or an empty list if the
+ * collection was not excluded.
  *
  * @examples
  *