From 44b8b9520e9f88097a075d6046d7eef6572b2585 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Fri, 9 Aug 2024 10:14:00 -0500 Subject: [PATCH] Bundle fixes and Morphir Mill Build extensions (#660) * Refine how we can lookup and index on many Distributions * Add additional methods around making Distributions and getting Libs * Add ErrorReason type * Upgrade to mill 0.11.10 * Setting up MorphirModule mill task to help use mill to support dependencies * Fix typo * Improve mill support for MorphirElmModules * produce artifacts with dist * Refining how we define MorphirModules in mill * Change package name for morphir plugin from starting with millbuild * Formatted build files * Adding additional projects for build * Adding additional projects for morphir-elm style build * Trigger make in other modules as a result of making a dependent module * Get dependencies working for morphir example projects * Working to make runtime tests auto build morphir models * Ensure MorphirElmModules are built before test is run * Install morphir-elm * Use setup-node --- .github/workflows/ci.yml | 39 +++ .mill-version | 2 +- .nvmrc | 2 +- build.sc | 79 +++++- .../src/millbuild/jsruntime/JsModule.scala | 33 +++ .../src/millbuild/jsruntime/JsRuntime.scala | 90 +++++++ mill-build/src/millbuild/util/helpers.scala | 43 ++++ .../org/finos/millmorphir/MorphirModule.scala | 229 ++++++++++++++++++ .../finos/millmorphir/api/ArtifactRef.scala | 50 ++++ .../org/finos/millmorphir/api/MakeArgs.scala | 28 +++ .../finos/millmorphir/api/MakeInputs.scala | 8 + .../finos/millmorphir/api/MakeOutputs.scala | 10 + .../finos/millmorphir/api/MakeResult.scala | 15 ++ .../api/MorphirProjectConfig.scala | 13 + .../src/org/finos/millmorphir/api/Named.scala | 10 + .../millmorphir/elm/MorphirElmModule.scala | 14 ++ .../finos/morphir/runtime/Distributions.scala | 2 +- .../runtime/quick/GatherReferences.scala | 4 +- .../finos/morphir/runtime/quick/Store.scala | 2 +- .../org/finos/morphir/error/ErrorReason.scala | 50 ++++ .../ir/distribution/Distribution.scala | 145 ++++++++++- .../ir/distribution/DistributionSpec.scala | 10 + .../MorphirBundlePlatformSpecific.scala | 2 +- project/deps.sc | 5 + 24 files changed, 859 insertions(+), 26 deletions(-) create mode 100644 mill-build/src/millbuild/jsruntime/JsModule.scala create mode 100644 mill-build/src/millbuild/jsruntime/JsRuntime.scala create mode 100644 mill-build/src/millbuild/util/helpers.scala create mode 100644 mill-build/src/org/finos/millmorphir/MorphirModule.scala create mode 100644 mill-build/src/org/finos/millmorphir/api/ArtifactRef.scala create mode 100644 mill-build/src/org/finos/millmorphir/api/MakeArgs.scala create mode 100644 mill-build/src/org/finos/millmorphir/api/MakeInputs.scala create mode 100644 mill-build/src/org/finos/millmorphir/api/MakeOutputs.scala create mode 100644 mill-build/src/org/finos/millmorphir/api/MakeResult.scala create mode 100644 mill-build/src/org/finos/millmorphir/api/MorphirProjectConfig.scala create mode 100644 mill-build/src/org/finos/millmorphir/api/Named.scala create mode 100644 mill-build/src/org/finos/millmorphir/elm/MorphirElmModule.scala create mode 100644 morphir/src/org/finos/morphir/error/ErrorReason.scala create mode 100644 morphir/tests/test/src/org/finos/morphir/ir/distribution/DistributionSpec.scala diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13674d4f5..5c4353f40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + + - name: Install morphir-elm + run: | + npm install -g morphir-elm + - name: Setup Scala and Java uses: actions/setup-java@v4 with: @@ -98,6 +108,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + + - name: Install morphir-elm + run: | + npm install -g morphir-elm + - name: Setup Scala and Java uses: actions/setup-java@v4 with: @@ -144,6 +164,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + + - name: Install morphir-elm + run: | + npm install -g morphir-elm + - name: Install libuv run: sudo apt-get update && sudo apt-get install -y libuv1-dev @@ -257,6 +287,15 @@ jobs: with: fetch-depth: 0 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + + - name: Install morphir-elm + run: | + npm install -g morphir-elm + - uses: actions/setup-java@v4 with: java-version: "11" diff --git a/.mill-version b/.mill-version index f677acceb..1a775473e 100644 --- a/.mill-version +++ b/.mill-version @@ -1 +1 @@ -0.11.11 \ No newline at end of file +0.11.11 diff --git a/.nvmrc b/.nvmrc index d5a159609..1a2f5bd20 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -20.10.0 +lts/* \ No newline at end of file diff --git a/build.sc b/build.sc index 63e6be772..4d9528581 100644 --- a/build.sc +++ b/build.sc @@ -1,3 +1,4 @@ +import mill.testrunner.TestResult import mill.scalalib.publish.PublishInfo import $meta._ import $ivy.`de.tototec::de.tobiasroeser.mill.integrationtest::0.7.1` @@ -11,6 +12,9 @@ import de.tobiasroeser.mill.integrationtest._ import io.kipp.mill.ci.release.CiReleaseModule import millbuild._ import millbuild.crossplatform._ +import millbuild.jsruntime._ +import org.finos.millmorphir._ +import org.finos.millmorphir.elm._ import millbuild.settings._ import mill._, mill.scalalib._, mill.scalajslib._, mill.scalanativelib._, scalafmt._ import mill.scalajslib.api.ModuleKind @@ -67,11 +71,12 @@ trait MorphirPublishModule extends CiReleaseModule with JavaModule with Mima { ) } -object morphir extends Cross[MorphirModule](buildSettings.scala.crossScalaVersions) { + +object morphir extends Cross[MorphirCrossModule](buildSettings.scala.crossScalaVersions) { object build extends Module { object integration extends Module { - object `mill-morphir-elm` extends Cross[MillMorphirElmPlugin](MillVersions.all) - trait MillMorphirElmPlugin + object `mill-morphir-elm` extends Cross[morphirlibElmPlugin](MillVersions.all) + trait morphirlibElmPlugin extends Cross.Module[String] with ScalaModule with ScalafmtModule @@ -139,7 +144,7 @@ object morphir extends Cross[MorphirModule](buildSettings.scala.crossScalaVersio } } -trait MorphirModule extends Cross.Module[String] with CrossPlatform { morphir => +trait MorphirCrossModule extends Cross.Module[String] with CrossPlatform { morphir => import DevMode._ val workspaceDir = millbuild.build.millSourcePath @@ -177,6 +182,7 @@ trait MorphirModule extends Cross.Module[String] with CrossPlatform { morphir => trait Shared extends MorphirCommonCrossModule with MorphirPublishModule { def ivyDeps = super.ivyDeps() ++ Agg( + Deps.org.`scala-lang`.modules.`scala-collection-contrib`, Deps.com.beachape.enumeratum, Deps.com.lihaoyi.fansi, Deps.com.lihaoyi.geny, @@ -351,8 +357,25 @@ trait MorphirModule extends Cross.Module[String] with CrossPlatform { morphir => def platformSpecificModuleDeps = Seq(morphir, morphir.interop.zio.json) } + trait RuntimeTests extends TestModule.ZioTest { + def morphirTestSources = T.sources { + examples.`morphir-elm-projects`.`evaluator-tests`.distOutputDirs() + } + + def morphirTestSourceFiles = T { + Lib.findSourceFiles(morphirTestSources(), Seq("json")).collect { + case path if path.last.startsWith("morphir-") => PathRef(path) + } + } + + override protected def testTask(args: Task[Seq[String]], globSelectors: Task[Seq[String]]): Task[(String, Seq[TestResult])] = T.task { + val _ = morphirTestSourceFiles() + super.testTask(args, globSelectors)() + } + } + object jvm extends Shared with MorphirJVMModule { - object test extends ScalaTests with TestModule.ZioTest { + object test extends ScalaTests with RuntimeTests { def ivyDeps = Agg( Deps.com.lihaoyi.`os-lib`, Deps.com.lihaoyi.sourcecode, @@ -364,7 +387,7 @@ trait MorphirModule extends Cross.Module[String] with CrossPlatform { morphir => } object js extends Shared with MorphirJSModule { - object test extends ScalaJSTests with TestModule.ZioTest { + object test extends ScalaJSTests with RuntimeTests { def ivyDeps = Agg(Deps.dev.zio.`zio-test`, Deps.dev.zio.`zio-test-sbt`) def moduleDeps = super.moduleDeps ++ Agg(testing.zio.js) def moduleKind = ModuleKind.CommonJSModule @@ -372,7 +395,7 @@ trait MorphirModule extends Cross.Module[String] with CrossPlatform { morphir => } object native extends Shared with MorphirNativeModule { - object test extends ScalaNativeTests with TestModule.ZioTest { + object test extends ScalaNativeTests with RuntimeTests { def ivyDeps = Agg(Deps.dev.zio.`zio-test`, Deps.dev.zio.`zio-test-sbt`) def moduleDeps = super.moduleDeps ++ Agg(testing.zio.native) } @@ -490,6 +513,48 @@ trait MorphirModule extends Cross.Module[String] with CrossPlatform { morphir => } } +// The following modules are morphir-elm modules that have been setup to be built using mill +// Morphir Elm Projects/Modules: +object examples extends Module { + object `morphir-elm-projects` extends Module { + object finance extends MorphirElmModule + + object `evaluator-tests` extends MorphirElmModule + object `unit-test-framework` extends Module { + object `example-project` extends MorphirElmModule { + def morphirModuleDeps = Seq(`morphir-elm`.sdks.`morphir-unit-test`) + } + + object `example-project-tests` extends MorphirElmModule { + def morphirModuleDeps = Seq( + `morphir-elm`.sdks.`morphir-unit-test`, + `example-project` + ) + } + object `example-project-tests-incomplete` extends MorphirElmModule { + def morphirModuleDeps = Seq( + `morphir-elm`.sdks.`morphir-unit-test`, + `example-project` + ) + } + object `example-project-tests-passing` extends MorphirElmModule { + def morphirModuleDeps = Seq( + `morphir-elm`.sdks.`morphir-unit-test`, + `example-project` + ) + } + } + } +} + +object `morphir-elm` extends Module { + object sdks extends Module { + object `morphir-unit-test` extends MorphirElmModule + } +} + +// The following section contains aliases used to simplify build tasks + object MyAliases extends Aliases { def fmt = alias("mill.scalalib.scalafmt.ScalafmtModule/reformatAll __.sources") def checkfmt = alias("mill.scalalib.scalafmt.ScalafmtModule/checkFormatAll __.sources") diff --git a/mill-build/src/millbuild/jsruntime/JsModule.scala b/mill-build/src/millbuild/jsruntime/JsModule.scala new file mode 100644 index 000000000..8b29daeb1 --- /dev/null +++ b/mill-build/src/millbuild/jsruntime/JsModule.scala @@ -0,0 +1,33 @@ +package millbuild.jsruntime + +import mill._ +import mill.define.Segment +import mill.scalalib._ + +trait JsModule extends Module { + def artifactName: Target[String] = T(millSourcePath.last) + def segments = T(millModuleSegments.value.collect { case Segment.Label(s) => s }) + def jsRunnerExecutable = T(PathRef(os.Path(JsRuntime.NodeJs.executable))) + def jsPackageManagerRunner: Target[String] = T("npx") + def jsPackageManagerCmd: Target[String] = T("npm") + + /** + * The folders containing all source files fed into the compiler + */ + def allSources: T[Seq[PathRef]] = T(sources() ++ generatedSources()) + + /** + * Folders containing source files that are generated rather than hand-written; these files can be generated in this + * target itself, or can refer to files generated from other targets + */ + def generatedSources: T[Seq[PathRef]] = T(Seq.empty[PathRef]) + def sources = T.sources(millSourcePath / "src") + + /** + * All individual source files fed into the compiler/tooling. + */ + def allSourceFiles: T[Seq[PathRef]] = T { + Lib.findSourceFiles(allSources(), Seq("js", "ts")).map(PathRef(_)) + } + +} diff --git a/mill-build/src/millbuild/jsruntime/JsRuntime.scala b/mill-build/src/millbuild/jsruntime/JsRuntime.scala new file mode 100644 index 000000000..35c6ec93c --- /dev/null +++ b/mill-build/src/millbuild/jsruntime/JsRuntime.scala @@ -0,0 +1,90 @@ +package millbuild.jsruntime +import scala.util.Properties.isWin +import mill.api._ +import mill.util.Jvm + +import os.SubProcess + +trait JsRuntime { + import JsRuntime._ + def toolName: String + + def findTool(name: String): String = + whereIs(name) + + def executable: String = + findTool(toolName) + + def runSubprocess(entryPoint: String, args: Seq[String], envArgs: Map[String, String], workingDir: os.Path)(implicit + ctx: Ctx + ): Unit = { + val commandArgs = Vector(executable) ++ args + println(s"CommandArgs: $commandArgs") + val process: SubProcess = Jvm.spawnSubprocessWithBackgroundOutputs( + commandArgs, + envArgs, + workingDir, + backgroundOutputs = None + ) + println(s"Createds process: ${process}") + + val shutdownHook = new Thread("subprocess-shutdown") { + override def run(): Unit = { + System.err.println(s"Host executable for $toolName shutdown. Forcefully destroying subprocess ...") + process.destroy() + } + } + Runtime.getRuntime().addShutdownHook(shutdownHook) + try + process.waitFor() + catch { + case e: InterruptedException => + System.err.println("Interrupted. Forcefully destroying subprocess ...") + process.destroy() + // rethrow + throw e + } finally + Runtime.getRuntime().removeShutdownHook(shutdownHook) + if (process.exitCode() == 0) () + else throw new Exception("Interactive Subprocess Failed (exit code " + process.exitCode() + ")") + + } +} + +object JsRuntime { + case object NodeJs extends JsRuntime { + def toolName: String = "node" + } + + def whereIs(name: String) = { + val commandArgs = + if (isWin) { + Vector("where", name) + } else { + Vector("whereis", name) + } + + val result = os.proc(commandArgs).call() + + val location = + if (isWin) { + result.out.lines().headOption.flatMap(_.trim().split(" ").headOption) + } else { + + result.out.lines().headOption.flatMap { line => + val parts = line.trim().split(" ") + parts match { + case Array(_, loc, _*) => Some(loc) + case _ => None + } + + } + + } + + location match { + case None => throw new Exception(s"Failed to locate tool: $name") + case Some(loc) => loc + } + } +} diff --git a/mill-build/src/millbuild/util/helpers.scala b/mill-build/src/millbuild/util/helpers.scala new file mode 100644 index 000000000..3abbd31cb --- /dev/null +++ b/mill-build/src/millbuild/util/helpers.scala @@ -0,0 +1,43 @@ +package millbuild.util +import scala.util.Properties.isWin + +object ProcessHelper { + def whereIs(name: String) = { + val commandArgs = + if (isWin) { + Vector("where", name) + } else { + Vector("whereis", name) + } + + val result = os.proc(commandArgs).call() + + val location = + if (isWin) { + result.out.lines().headOption.flatMap(_.trim().split(" ").headOption) + } else { + + result.out.lines().headOption.flatMap { line => + val parts = line.trim().split(" ") + parts match { + case Array(_, loc, _*) => Some(loc) + case _ => None + } + } + } + + location match { + case None => throw new Exception(s"Failed to locate tool: $name") + case Some(loc) => loc + } + } +} + +object Collections { + implicit class SeqOps[+A](private val self: Seq[A]) extends AnyVal { + def appendIf[A1 >: A](cond: Boolean)(items: A1*) = if (cond) self ++ items else self + def appendWhen[A1 >: A](cond: => Boolean)(items: A1*) = if (cond) self ++ items else self + + def when[A1 >: A](cond: Boolean)(items: A1*) = if (cond) items else Seq.empty[A1] + } +} diff --git a/mill-build/src/org/finos/millmorphir/MorphirModule.scala b/mill-build/src/org/finos/millmorphir/MorphirModule.scala new file mode 100644 index 000000000..234dc043d --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/MorphirModule.scala @@ -0,0 +1,229 @@ +package org.finos.millmorphir + +import millbuild.util.Collections._ +import millbuild.util.ProcessHelper +import millbuild.jsruntime.JsRuntime +import org.finos.millmorphir.api._ +import mill._ +import mill.define.{Segment, Segments} +import mill.api.JsonFormatters._ +import mill.scalalib._ +import mill.scalalib.internal.ModuleUtils +import upickle.default._ + +trait MorphirModule extends Module { self => + + def clean() = T.command { + var pendingDelete = List(morphirProjectDirResolved().path / "morphir-hashes.json") + pendingDelete.foreach { path => + if (os.exists(path)) { + os.remove.all(path) + } + } + } + + def dist:T[Set[ArtifactRef]] = T { + val incrementalBuildFiles = incrementalMakeSourceFiles() + val outputs = make() + + val distPath = distFolder().path + if (!os.exists(distPath)) { + os.makeDir.all(distPath) + } + + outputs.artifacts.map { artifact => + val path = artifact.path + val targetDir = distPath + if (!os.exists(targetDir)) { + os.makeDir.all(targetDir) + } + + val targetPath = targetDir / path.last + + T.ctx().log.debug(s"Copying ${path} to $targetPath") + try { + if(os.exists(path) && path != targetPath) { + os.copy.over(path, targetPath) + } + Option(artifact.withPath(targetPath)) + } catch { + case e: Exception => + T.ctx().log.error(s"Failed to copy $path to $targetPath") + T.ctx().log.error(e.toString) + None + } + + }.collect { case Some(artifact) => artifact } + } + + final def distOutputDirs = T.sources ( + dist().map{ case artifactRef:ArtifactRef => + val path = artifactRef.path / os.up + PathRef(path) + }.toSeq + ) + + def distFolder: Target[PathRef] = T { + //PathRef(morphirProjectDirResolved().path / "dist") + morphirProjectDirResolved() + } + + def incrementalMakeSources = T.sources { + millSourcePath + } + + def incrementalMakeSourceFiles = T{ + val sourceFileNames = Set("morphir-hashes.json", "morphir-ir.json") + for { + source <- incrementalMakeSources() + sourceFileName <- sourceFileNames + sourceFile = source.path / sourceFileName + if os.exists(sourceFile) + } yield PathRef(sourceFile) + } + + /// Use indentation in the generated JSON file. + def indentJson: Target[Boolean] = T(false) + + def morphirCommand: Target[String] = T("morphir") + def morphirProjectDir: Target[PathRef] = T.source(PathRef(millSourcePath)) + final def morphirProjectDirResolved: Target[PathRef] = T { + PathRef(makeArgs().projectDir) + } + def morphirHashesPath: Target[PathRef] = T { + PathRef(morphirProjectDir().path / "morphir-hashes.json") + } + + def morphirHashesContent: Target[Option[ujson.Value]] = T { + if (os.exists(morphirProjectDirResolved().path)) { + Option(ujson.read(os.read(morphirHashesPath().path))) + } else { + None + } + } + + def morphirProjectSources = T.sources { + millSourcePath + } + + def morphirProjectSourceFileNames = T { + Set("morphir.json") + } + + def morphirProjectSourceFiles = T.sources { + for { + source <- morphirProjectSources() + sourceFileName <- morphirProjectSourceFileNames() + sourceFile = source.path / sourceFileName + if os.exists(sourceFile) + } yield PathRef(sourceFile) + } + + def sources = T.sources { Seq(PathRef(millSourcePath / "src")) } + + def allSourceFiles: T[Seq[PathRef]] = T { + sources().map(_.path).flatMap(os.walk(_).filter(_.toIO.isFile)).map(PathRef(_)) + } + + def morphirIncrementalBuildSourceFiles = T.sources { + Seq(PathRef(morphirProjectDir().path / "morphir-hashes.json")) + } + + def morphirProjectConfig: Target[MorphirProjectConfig] = T { + val morphirProjectFile = morphirProjectDir().path / "morphir.json" + if (os.exists(morphirProjectFile)) { + read[MorphirProjectConfig](os.read(morphirProjectFile)) + } else { + throw new Exception(s"morphir.json file not found, looked for it at ${morphirProjectFile}.") + } + } + + def makeCommandRunner: Target[String] = T { + ProcessHelper.whereIs(morphirCommand()) + } + + def morphirIrFilename = T("morphir-ir.json") + + def moduleId = T { + millModuleSegments.render + } + + def make: T[MakeOutputs] = T { + val makeResult = morphirMake() + val artifacts:Set[ArtifactRef] = Set(ArtifactRef.morphirIR(makeResult.irFilePath, "morphir", "ir")) ++ makeResult.morphirHashesPath.map { path => + Seq(ArtifactRef.morphirHashes(path, "morphir", "hashes", "incremental")) + }.getOrElse(Seq.empty).toSet + + MakeOutputs(moduleId(), artifacts) + } + + def makeArgs: Task[MakeArgs] = T.task { + MakeArgs( + projectDir = morphirProjectDir().path, + output = T.dest / morphirIrFilename(), + indentJson = indentJson(), + typesOnly = typesOnly(), + fallbackCli = None + ) + } + + + + /** + * The direct dependencies of this module. This is meant to be overridden to add dependencies. To read the value, you + * should use [[morphirModuleDepsChecked]] instead, which uses a cached result which is also checked to be free of + * cycles. + * @see + * [[morphirModuleDepsChecked]] + */ + def morphirModuleDeps: Seq[MorphirModule] = Seq.empty + + /** + * Same as [[morphirModuleDeps]], but checked to not contain cycles. Prefer this over using [[moduleDeps]] directly. + */ + final def morphirModuleDepsChecked: Seq[MorphirModule] = { + recMorphirModuleDeps + morphirModuleDeps + } + + /** Should only be called from [[moduleDepsChecked]] */ + private lazy val recMorphirModuleDeps: Seq[MorphirModule] = + ModuleUtils.recursive[MorphirModule]( + (millModuleSegments ++ Seq(Segment.Label("morphirModuleDeps"))).render, + this, + _.morphirModuleDeps + ) + + /** The direct and indirect dependencies of this module */ + def recursiveMorphirModuleDeps: Seq[MorphirModule] = + recMorphirModuleDeps + + /** + * Like `recursiveMorphirModuleDeps`, but includes this module itself as well. + */ + def transitiveMorphirModuleDeps: Seq[MorphirModule] = Seq(this) ++ recursiveMorphirModuleDeps + + def upstreamMakeOutput = T { + T.traverse(recursiveMorphirModuleDeps.distinct)(_.dist) + } + + def morphirMake: Target[MakeResult] = T { + val makeArgs: MakeArgs = self.makeArgs() + val cli = makeCommandRunner() + + val _ = upstreamMakeOutput() + val _ = allSourceFiles() + val commandArgs = makeArgs.toCommandArgs(cli) + val workingDir = makeArgs.projectDir + val destPath = makeArgs.output + util.Jvm.runSubprocess(commandArgs, T.ctx().env, workingDir) + val hashesPath = morphirHashesPath() + val hashesPathFinal = + if (os.exists(hashesPath.path)) Option(hashesPath) else None + MakeResult(makeArgs, PathRef(destPath), commandArgs, workingDir, morphirHashesPath = hashesPathFinal) + } + + /// Only include type information in the IR, no values. + def typesOnly: Target[Boolean] = T(false) + +} diff --git a/mill-build/src/org/finos/millmorphir/api/ArtifactRef.scala b/mill-build/src/org/finos/millmorphir/api/ArtifactRef.scala new file mode 100644 index 000000000..cfaaaea55 --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/api/ArtifactRef.scala @@ -0,0 +1,50 @@ +package org.finos.millmorphir.api +import mill._ +import upickle.default.{ReadWriter => RW, readwriter, macroRW} + +final case class ArtifactRef( + pathRef: PathRef, + artifactType: ArtifactType, + tags: Set[String] = Set.empty +) { self => + def addTag(tag: String): ArtifactRef = self.copy(tags = tags + tag) + def addTags(tags: String*): ArtifactRef = self.copy(tags = self.tags ++ tags) + def path: os.Path = pathRef.path + def withPath(path: os.Path): ArtifactRef = self.copy(pathRef = PathRef(path)) +} + +object ArtifactRef { + implicit val jsonFormatter: RW[ArtifactRef] = macroRW + def morphirIR(pathRef: PathRef, tags: String*): ArtifactRef = + ArtifactRef(pathRef, ArtifactType.MorphirIR, Set(tags: _*)) + def morphirHashes(pathRef: PathRef, tags: String*): ArtifactRef = + ArtifactRef(pathRef, ArtifactType.MorphirHashes, Set(tags: _*)) + def custom(name: String, pathRef: PathRef, tags: String*): ArtifactRef = + ArtifactRef(pathRef, ArtifactType.Custom(name), Set(tags: _*)) +} + +sealed abstract class ArtifactType(val tag: String) extends Product with Serializable { + override def toString: String = tag +} + +object ArtifactType { + implicit val jsonFormatter: RW[ArtifactType] = readwriter[String].bimap(_.tag, fromTag) + + def fromTag(tag: String): ArtifactType = tag match { + case MorphirIR.tag => MorphirIR + case MorphirHashes.tag => MorphirHashes + case _ if tag.startsWith("custom:") => Custom(tag) + case _ => throw new IllegalArgumentException(s"Unknown artifact type: $tag") + } + + case object MorphirIR extends ArtifactType("morphir-ir") { self => + implicit val jsonFormatter: RW[MorphirIR.type] = readwriter[String].bimap(_.tag, _ => self) + } + case object MorphirHashes extends ArtifactType("morphir-hashes") { self => + implicit val jsonFormatter: RW[MorphirHashes.type] = readwriter[String].bimap(_.tag, _ => self) + } + case class Custom(name: String) extends ArtifactType(s"custom:$name") + object Custom { + implicit val jsonFormatter: RW[Custom] = readwriter[String].bimap(_.tag, Custom(_)) + } +} diff --git a/mill-build/src/org/finos/millmorphir/api/MakeArgs.scala b/mill-build/src/org/finos/millmorphir/api/MakeArgs.scala new file mode 100644 index 000000000..3ea30642a --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/api/MakeArgs.scala @@ -0,0 +1,28 @@ +package org.finos.millmorphir.api + +import mill.api.JsonFormatters._ +import millbuild.util.Collections._ + +final case class MakeArgs( + projectDir: os.Path, + output: os.Path, + indentJson: Boolean, + typesOnly: Boolean, + fallbackCli: Option[Boolean] +) { self => + def useFallbackCli: Boolean = self.fallbackCli.getOrElse(false) + + def toCommandArgs: Seq[String] = + Seq("make") + .appendWhen(Option(projectDir).nonEmpty)("--project-dir", projectDir.toString()) + .appendWhen(Option(output).nonEmpty)("--output", output.toString()) + .appendIf(indentJson)("--indent-json") + .appendIf(typesOnly)("--types-only") + .appendIf(useFallbackCli)("--fallback-cli") + + def toCommandArgs(cli: String): Seq[String] = Seq(cli) ++ toCommandArgs +} + +object MakeArgs { + implicit val jsonFormatter: upickle.default.ReadWriter[MakeArgs] = upickle.default.macroRW +} diff --git a/mill-build/src/org/finos/millmorphir/api/MakeInputs.scala b/mill-build/src/org/finos/millmorphir/api/MakeInputs.scala new file mode 100644 index 000000000..ee5a4dc29 --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/api/MakeInputs.scala @@ -0,0 +1,8 @@ +package org.finos.millmorphir.api +import mill.api._ +import mill.api.JsonFormatters._ + +final case class MakeInputs(sourceFiles: Seq[PathRef], incrementalBuildFiles: Seq[PathRef]) +object MakeInputs { + implicit val jsonFormatter: upickle.default.ReadWriter[MakeInputs] = upickle.default.macroRW +} diff --git a/mill-build/src/org/finos/millmorphir/api/MakeOutputs.scala b/mill-build/src/org/finos/millmorphir/api/MakeOutputs.scala new file mode 100644 index 000000000..6b434e663 --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/api/MakeOutputs.scala @@ -0,0 +1,10 @@ +package org.finos.millmorphir.api + +final case class MakeOutputs(moduleId:String, artifacts:Set[ArtifactRef]) { + def addArtifact(artifact: ArtifactRef): MakeOutputs = MakeOutputs(moduleId, artifacts + artifact) + def addArtifacts(artifacts: ArtifactRef*): MakeOutputs = MakeOutputs(moduleId, this.artifacts ++ artifacts) +} + +object MakeOutputs { + implicit val jsonFormatter: upickle.default.ReadWriter[MakeOutputs] = upickle.default.macroRW +} diff --git a/mill-build/src/org/finos/millmorphir/api/MakeResult.scala b/mill-build/src/org/finos/millmorphir/api/MakeResult.scala new file mode 100644 index 000000000..77e6bafb2 --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/api/MakeResult.scala @@ -0,0 +1,15 @@ +package org.finos.millmorphir.api + +import mill.PathRef +import mill.api.JsonFormatters._ + +case class MakeResult( + makeArgs: MakeArgs, + irFilePath: PathRef, + commandArgs: Seq[String], + workingDir: os.Path, + morphirHashesPath: Option[PathRef] = None +) +object MakeResult { + implicit val jsonFormatter: upickle.default.ReadWriter[MakeResult] = upickle.default.macroRW +} diff --git a/mill-build/src/org/finos/millmorphir/api/MorphirProjectConfig.scala b/mill-build/src/org/finos/millmorphir/api/MorphirProjectConfig.scala new file mode 100644 index 000000000..0293f218b --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/api/MorphirProjectConfig.scala @@ -0,0 +1,13 @@ +package org.finos.millmorphir.api + +final case class MorphirProjectConfig( + name: String, + sourceDirectory: String, + exposedModules: List[String], + dependencies: List[String], + localDependencies: List[String] +) + +object MorphirProjectConfig { + implicit val jsonFormatter: upickle.default.ReadWriter[MorphirProjectConfig] = upickle.default.macroRW +} diff --git a/mill-build/src/org/finos/millmorphir/api/Named.scala b/mill-build/src/org/finos/millmorphir/api/Named.scala new file mode 100644 index 000000000..4f735c549 --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/api/Named.scala @@ -0,0 +1,10 @@ +package org.finos.millmorphir.api + +final case class Named[+A](name:String, value:A) { + def map[B](f: A => B): Named[B] = Named(name, f(value)) + def flatMap[B](f: A => Named[B]): Named[B] = f(value) +} + +object Named { + implicit def jsonFormatter[A: upickle.default.ReadWriter]: upickle.default.ReadWriter[Named[A]] = upickle.default.macroRW +} \ No newline at end of file diff --git a/mill-build/src/org/finos/millmorphir/elm/MorphirElmModule.scala b/mill-build/src/org/finos/millmorphir/elm/MorphirElmModule.scala new file mode 100644 index 000000000..0017bc9a9 --- /dev/null +++ b/mill-build/src/org/finos/millmorphir/elm/MorphirElmModule.scala @@ -0,0 +1,14 @@ +package org.finos.millmorphir.elm +import mill._ +import mill.scalalib._ +import org.finos.millmorphir.MorphirModule + +trait MorphirElmModule extends MorphirModule { + def allSourceFiles: T[Seq[PathRef]] = T { + sources().map(_.path).flatMap(os.walk(_).filter(_.ext == "elm")).map(PathRef(_)) + } + + def morphirProjectSourceFileNames = T { + super.morphirProjectSourceFileNames() ++ Set("elm.json") + } +} diff --git a/morphir/runtime/src/org/finos/morphir/runtime/Distributions.scala b/morphir/runtime/src/org/finos/morphir/runtime/Distributions.scala index dcfe64d2d..37aa96f00 100644 --- a/morphir/runtime/src/org/finos/morphir/runtime/Distributions.scala +++ b/morphir/runtime/src/org/finos/morphir/runtime/Distributions.scala @@ -107,7 +107,7 @@ class Distributions(dists: Map[PackageName, Distribution.Lib]) { object Distributions { def apply(dists: Distribution*): Distributions = - new Distributions(Distribution.toLibsMap(dists: _*)) + new Distributions(Distribution.toLibsMapUnsafe(dists: _*)) def apply(dists: Map[PackageName, Distribution.Lib]): Distributions = new Distributions(dists) } diff --git a/morphir/runtime/src/org/finos/morphir/runtime/quick/GatherReferences.scala b/morphir/runtime/src/org/finos/morphir/runtime/quick/GatherReferences.scala index 275c13edc..362da70c1 100644 --- a/morphir/runtime/src/org/finos/morphir/runtime/quick/GatherReferences.scala +++ b/morphir/runtime/src/org/finos/morphir/runtime/quick/GatherReferences.scala @@ -29,7 +29,7 @@ object GatherReferences { globals.ctors.keys.foldLeft(ReferenceSet.empty)((acc, next) => acc.withConstructor(next)) def fromEntrySet(entrySet: ReferenceSet, dists: Distribution*): ReferenceSet = { - val mapped = Distribution.toLibsMap(dists: _*) + val mapped = Distribution.toLibsMapUnsafe(dists: _*) def f(known: Set[FQName], ref: FQName): Set[FQName] = { // if (depth > 100) throw new Exception(s"Still recursing on $next with known values ${known.toList.mkString("\n")}") @@ -50,7 +50,7 @@ object GatherReferences { } def fromDistributions(dists: Distribution*): ReferenceSet = - fromDistributionLibs(Distribution.toLibsMap(dists: _*)) + fromDistributionLibs(Distribution.toLibsMapUnsafe(dists: _*)) def fromDistributionLibs(libs: Map[PackageName, Lib]): ReferenceSet = libs.foldLeft(ReferenceSet.empty) { case (acc: ReferenceSet, (packageName: PackageName, lib: Lib)) => diff --git a/morphir/runtime/src/org/finos/morphir/runtime/quick/Store.scala b/morphir/runtime/src/org/finos/morphir/runtime/quick/Store.scala index e5621e0aa..558612ff0 100644 --- a/morphir/runtime/src/org/finos/morphir/runtime/quick/Store.scala +++ b/morphir/runtime/src/org/finos/morphir/runtime/quick/Store.scala @@ -42,7 +42,7 @@ final case class GlobalDefs( object GlobalDefs { def fromDistributions(dists: Distribution*): GlobalDefs = { - val libs: Map[PackageName, Lib] = Distribution.toLibsMap(dists: _*) + val libs: Map[PackageName, Lib] = Distribution.toLibsMapUnsafe(dists: _*) libs.foldLeft(native) { case (acc, (packageName, lib)) => createDefs(acc, packageName, lib.dependencies, lib.packageDef) } diff --git a/morphir/src/org/finos/morphir/error/ErrorReason.scala b/morphir/src/org/finos/morphir/error/ErrorReason.scala new file mode 100644 index 000000000..560ac229e --- /dev/null +++ b/morphir/src/org/finos/morphir/error/ErrorReason.scala @@ -0,0 +1,50 @@ +package org.finos.morphir.error + +import scala.annotation.tailrec + +sealed trait ErrorReason[+E] { self => + import ErrorReason._ + + def &&[E1 >: E](that: ErrorReason[E1]): ErrorReason[E1] = Both(self, that) + def ++[E1 >: E](that: ErrorReason[E1]): ErrorReason[E1] = + if (self eq Empty) that else if (that eq Empty) self else Then(self, that) + + /// Produces a list of all recoverable errors `E` in the `ErrorReason`. + final def failures: List[E] = + self.foldLeft(List.empty[E]) { case (z, Fail(v)) => + v :: z + } + .reverse + + final def foldLeft[Z](z: Z)(f: PartialFunction[(Z, ErrorReason[E]), Z]): Z = { + @tailrec + def loop(z0: Z, reason: ErrorReason[E], stack: List[ErrorReason[E]]): Z = { + val z = f.applyOrElse[(Z, ErrorReason[E]), Z](z0 -> reason, _._1) + reason match { + case Then(left, right) => loop(z, left, right :: stack) + case Both(left, right) => loop(z, left, right :: stack) + case _ if stack.nonEmpty => loop(z, stack.head, stack.tail) + case _ => z + } + } + if (self eq Empty) z + else loop(z, self, Nil) + + } + + // def flatMap[E2](f: E => ErrorCause[E2]):ErrorCause[E2] = self match { + // case Empty => + // } + // def map[E1](f: E => E1):ErrorCause[E1] = ??? +} + +object ErrorReason { + val empty: ErrorReason[Nothing] = Empty + def fail[E](error: E): ErrorReason[E] = Fail(error) + def both[E](left: ErrorReason[E], right: ErrorReason[E]): ErrorReason[E] = Both(left, right) + + case object Empty extends ErrorReason[Nothing] + final case class Fail[+E](error: E) extends ErrorReason[E] + final case class Both[+E](left: ErrorReason[E], right: ErrorReason[E]) extends ErrorReason[E] + final case class Then[+E](left: ErrorReason[E], right: ErrorReason[E]) extends ErrorReason[E] +} diff --git a/morphir/src/org/finos/morphir/ir/distribution/Distribution.scala b/morphir/src/org/finos/morphir/ir/distribution/Distribution.scala index 184d67737..c4d88520b 100644 --- a/morphir/src/org/finos/morphir/ir/distribution/Distribution.scala +++ b/morphir/src/org/finos/morphir/ir/distribution/Distribution.scala @@ -7,8 +7,26 @@ import org.finos.morphir.ir.Type.Specification.TypeAliasSpecification import org.finos.morphir.ir.Type.Type.Reference import org.finos.morphir.ir.Type.UType import org.finos.morphir.ir.Value.{USpecification => UValueSpec, Definition => ValueDefinition} - -sealed trait Distribution +import scala.collection.immutable.MultiDict +import scala.annotation.tailrec +import org.finos.morphir.ir.distribution.Distribution.RepeatedPackages.Allowed +import org.finos.morphir.ir.distribution.Distribution.RepeatedPackages.NotAllowed + +sealed trait Distribution { self => + import Distribution._ + + /// Get all distributions contained in this distribution. + /// Note: This will expand bundles but not dependencies. + def allDistributions: List[Distribution] = { + @tailrec + def loop(pending: List[Distribution], acc: List[Distribution]): List[Distribution] = pending match { + case Nil => acc + case (lib @ Library(_, _, _)) :: rest => loop(rest, lib :: acc) + case (bundle @ Bundle(_)) :: rest => loop(bundle.toLibraries ++ rest, acc) + } + loop(self :: Nil, List.empty) + } +} object Distribution { final case class Lib( dependencies: Map[PackageName, UPackageSpecification], @@ -20,6 +38,9 @@ object Distribution { def lookupTypeDefinition(qName: QName): Option[UTypeDef] = packageDef.lookupModuleDefinition(qName.modulePath).flatMap(_.lookupTypeDefinition(qName.localName)) + + private[Distribution] def toLibrary(packageName: PackageName): Library = + Library(packageName, self.dependencies, self.packageDef) } final case class Bundle( @@ -30,6 +51,10 @@ object Distribution { case bundle: Bundle => Bundle(libraries ++ bundle.libraries) case library: Library => Bundle(libraries + (library.packageName -> library.toLib)) } + + private[Distribution] def toLibraries: List[Library] = self.libraries.map { + case (packageName, lib) => lib.toLibrary(packageName) + }.toList } final case class Library( @@ -86,7 +111,23 @@ object Distribution { ): Bundle = Bundle(Map(packageName -> Lib(dependencies, packageDef))) def toBundle(packageName: PackageName, lib: Lib): Bundle = Bundle(Map(packageName -> lib)) - def toBundle(dists: Distribution*): Bundle = Bundle(toLibsMap(dists: _*)) + + def toBundleUnsafe(distributions: Distribution*): Bundle = toBundleUnsafe(BundleSettings.default, distributions: _*) + def toBundleUnsafe(settings: BundleSettings, distributions: Distribution*): Bundle = settings.RepeatedPackages match { + case RepeatedPackages.Allowed => + val map = distributions.flatMap { + case (library: Library) => List(library.packageName -> library.toLib) + case (bundle: Bundle) => bundle.libraries.toList + }.toMap + Bundle(map) + case RepeatedPackages.NotAllowed => + val lookup = toLookup(distributions: _*) + val repeatedPackages = lookup.repeatedPackages + if (repeatedPackages.nonEmpty) { + throw new BundlingError.MultiplePackagesWithSameNameDetected(repeatedPackages) + } + Bundle(lookup.toMultiDict.toMap) + } def toLibrary( packageName: PackageName, @@ -95,13 +136,93 @@ object Distribution { ): Library = Library(packageName, dependencies, packageDef) def toLibrary(packageName: PackageName, lib: Lib): Library = Library(packageName, lib.dependencies, lib.packageDef) - def toLibraries(dists: Distribution*): List[Library] = toLibsMap(dists: _*) - .map { case (packageName, lib) => toLibrary(packageName, lib) } - .toList - - def toLibsMap(dists: Distribution*): Map[PackageName, Lib] = - dists.flatMap { - case (library: Library) => List(library.packageName -> library.toLib) - case (bundle: Bundle) => bundle.libraries.toList - }.toMap + + def toLibraries(distributions: Distribution*): List[Library] = { + @tailrec + def loop(pending: List[Distribution], acc: List[Library]): List[Library] = pending match { + case Nil => acc + case (lib @ Library(_, _, _)) :: rest => loop(rest, lib :: acc) + case Bundle(libraries) :: rest => + val newLibs = libraries.map { case (packageName, lib) => lib.toLibrary(packageName) }.toList + loop(rest, newLibs ++ acc) + } + loop(distributions.toList, List.empty) + } + + def toLookup(distributions: Distribution*): LibLookup = LibLookup.fromDistributions(distributions) + + def toLibsMapUnsafe(distributions: Distribution*): Map[PackageName, Lib] = + toLibsMapUnsafe(RepeatedPackages.NotAllowed, distributions: _*) + + def toLibsMapUnsafe(repeatedPackages: RepeatedPackages, distributions: Distribution*): Map[PackageName, Lib] = { + val lookup = toLookup(distributions: _*) + repeatedPackages match { + case Allowed => lookup.toMultiDict.toMap + case NotAllowed => + val lookup = toLookup(distributions: _*) + val repeatedPackages = lookup.repeatedPackages + if (repeatedPackages.nonEmpty) { + throw new BundlingError.MultiplePackagesWithSameNameDetected(repeatedPackages) + } + lookup.toMultiDict.toMap + } + } + + final case class LibLookup(toMultiDict: MultiDict[PackageName, Lib]) extends AnyVal { self => + def packageNames: scala.collection.Set[PackageName] = toMultiDict.keySet + def repeatedPackages: Set[PackageName] = + toMultiDict.sets.collect { case (packageName, libs) if libs.size > 1 => packageName }.toSet + } + + object LibLookup { + def apply(distributions: Distribution*): LibLookup = fromDistributions(distributions) + + def fromDistributions(distributions: Seq[Distribution]): LibLookup = { + def loop(pending: List[Distribution], acc: MultiDict[PackageName, Lib]): LibLookup = pending match { + case Nil => LibLookup(acc) + case Library(packageName, dependencies, packageDef) :: rest => + loop(rest, acc.add(packageName, Lib(dependencies, packageDef))) + case (bundle @ Bundle(_)) :: rest => loop(bundle.toLibraries ++ rest, acc) + } + loop(distributions.toList, MultiDict.empty) + } + } + + trait DistributionError + + sealed abstract class BundlingError(message: String) extends Exception(message) with DistributionError with Product + with Serializable + object BundlingError { + + def failWithMultiplePackagesWithSameNameDetected( + packageName: PackageName, + others: PackageName* + ): MultiplePackagesWithSameNameDetected = others match { + case Nil => MultiplePackagesWithSameNameDetected(Set(packageName)) + case _ => MultiplePackagesWithSameNameDetected(Set(packageName) ++ Set.from(others)) + } + + final case class MultiplePackagesWithSameNameDetected private[Distribution] (packages: Set[PackageName]) + extends BundlingError( + s"Multiple packages with the same name detected. Repeated packages are: ${packages.mkString(", ")}" + ) { self => + def +(other: PackageName): MultiplePackagesWithSameNameDetected = + MultiplePackagesWithSameNameDetected(self.packages + other) + def +:(other: PackageName): MultiplePackagesWithSameNameDetected = + MultiplePackagesWithSameNameDetected(self.packages + other) + + } + } + + final case class BundleSettings(RepeatedPackages: RepeatedPackages) + object BundleSettings { + val default: BundleSettings = BundleSettings(RepeatedPackages.NotAllowed) + } + + sealed abstract class RepeatedPackages extends Product with Serializable + object RepeatedPackages { + case object Allowed extends RepeatedPackages + case object NotAllowed extends RepeatedPackages + } + } diff --git a/morphir/tests/test/src/org/finos/morphir/ir/distribution/DistributionSpec.scala b/morphir/tests/test/src/org/finos/morphir/ir/distribution/DistributionSpec.scala new file mode 100644 index 000000000..8a37fa8f1 --- /dev/null +++ b/morphir/tests/test/src/org/finos/morphir/ir/distribution/DistributionSpec.scala @@ -0,0 +1,10 @@ +package org.finos.morphir +package ir +package distribution + +import org.finos.morphir.testing.MorphirBaseSpec +import zio.test._ + +object DistributionSpec extends MorphirBaseSpec { + def spec = suite("Distribution Spec")() +} diff --git a/morphir/tools/jvm-native/src/org/finos/morphir/service/MorphirBundlePlatformSpecific.scala b/morphir/tools/jvm-native/src/org/finos/morphir/service/MorphirBundlePlatformSpecific.scala index 474943c0f..05f4849b8 100644 --- a/morphir/tools/jvm-native/src/org/finos/morphir/service/MorphirBundlePlatformSpecific.scala +++ b/morphir/tools/jvm-native/src/org/finos/morphir/service/MorphirBundlePlatformSpecific.scala @@ -21,7 +21,7 @@ trait MorphirBundlePlatformSpecific { _ <- Console.printLine(s"\toutputPath: $outputPath") _ <- Console.printLine(s"\tirFiles: $irFiles") distributions <- ZIO.collectAll { irFiles.map { irFile => loadDistributionFromFileZIO(irFile.toString) } } - bundle <- ZIO.attempt { Distribution.toBundle(distributions: _*) } + bundle <- ZIO.attempt { Distribution.toBundleUnsafe(distributions: _*) } writtenPath <- writeDistributionToFileZIO(bundle, outputPath) _ <- Console.printLine(s"\tBundle Morphir IR file created: $writtenPath") _ <- Console.printLine("Bundle command executed") diff --git a/project/deps.sc b/project/deps.sc index 9a894f4b9..6bd40a692 100644 --- a/project/deps.sc +++ b/project/deps.sc @@ -152,6 +152,11 @@ object Deps { } case object org { case object `scala-lang` { + + case object modules { + val `scala-collection-contrib` = ivy"org.scala-lang.modules::scala-collection-contrib:0.3.0" + } + def `scala-compiler`(scalaVersion: String): Dep = if (scalaVersion.startsWith("3")) ivy"org.scala-lang::scala3-compiler:$scalaVersion" else ivy"org.scala-lang:scala-compiler:$scalaVersion"