diff --git a/.changeset/nervous-nails-suffer.md b/.changeset/nervous-nails-suffer.md new file mode 100644 index 0000000000..245f968d7d --- /dev/null +++ b/.changeset/nervous-nails-suffer.md @@ -0,0 +1,5 @@ +--- +"@inlang/plugin-i18next": minor +--- + +increase batching to 50 for i18n plugin diff --git a/.gitignore b/.gitignore index a651d5e5da..a746bc244e 100644 --- a/.gitignore +++ b/.gitignore @@ -375,3 +375,7 @@ inlang/source-code/paraglide/paraglide-solidstart/example/.solid *.h.ts.mjs **/vite.config.ts.timestamp-* **/vite.config.js.timestamp-* + + +# gitea test instance data +lix/packages/gitea \ No newline at end of file diff --git a/inlang/source-code/editor/src/pages/@host/@owner/@repository/+Page.tsx b/inlang/source-code/editor/src/pages/@host/@owner/@repository/+Page.tsx index 8c8dc82431..765234f962 100644 --- a/inlang/source-code/editor/src/pages/@host/@owner/@repository/+Page.tsx +++ b/inlang/source-code/editor/src/pages/@host/@owner/@repository/+Page.tsx @@ -45,8 +45,16 @@ export default function Page() { * is required to use the useEditorState hook. */ function TheActualPage() { - const { repo, currentBranch, project, projectList, routeParams, tourStep, lixErrors, languageTags } = - useEditorState() + const { + repo, + currentBranch, + project, + projectList, + routeParams, + tourStep, + lixErrors, + languageTags, + } = useEditorState() const [localStorage, setLocalStorage] = useLocalStorage() onMount(() => { @@ -165,12 +173,16 @@ function TheActualPage() { currentId="textfield" position="bottom-left" offset={{ x: 110, y: 144 }} - isVisible={tourStep() === "textfield" && messageCount() !== 0 && languageTags().length > 1} - ><> - - {(id) => { - return - }} + isVisible={ + tourStep() === "textfield" && messageCount() !== 0 && languageTags().length > 1 + } + > + <> + + + {(id) => { + return + }} diff --git a/inlang/source-code/editor/src/pages/@host/@owner/@repository/Layout.tsx b/inlang/source-code/editor/src/pages/@host/@owner/@repository/Layout.tsx index fb0998aa56..8ea3a86594 100644 --- a/inlang/source-code/editor/src/pages/@host/@owner/@repository/Layout.tsx +++ b/inlang/source-code/editor/src/pages/@host/@owner/@repository/Layout.tsx @@ -467,7 +467,8 @@ function Breadcrumbs() { * The menu to select the branch. */ function BranchMenu() { - const { activeBranch, setActiveBranch, branchNames, currentBranch } = useEditorState() + const { activeBranch, setActiveBranch, setBranchListEnabled, branchList, currentBranch } = + useEditorState() return ( - + setBranchListEnabled(true)}>
{/* branch icon from github */} @@ -497,15 +501,25 @@ function BranchMenu() { - - {(branch) => ( -
setActiveBranch(branch)}> - - {branch} - -
- )} - + Loading...} + > + + {(branch) => ( +
{ + setActiveBranch(branch) + setBranchListEnabled(false) // prevent refetching after selecting branch + }} + > + + {branch} + +
+ )} + + diff --git a/inlang/source-code/editor/src/pages/@host/@owner/@repository/State.tsx b/inlang/source-code/editor/src/pages/@host/@owner/@repository/State.tsx index 07055c8eae..e778dd3c4c 100644 --- a/inlang/source-code/editor/src/pages/@host/@owner/@repository/State.tsx +++ b/inlang/source-code/editor/src/pages/@host/@owner/@repository/State.tsx @@ -51,7 +51,7 @@ type EditorStateSchema = { * Fork status of the repository. */ - forkStatus: () => { ahead: number; behind: number; conflicts: boolean } + forkStatus: () => { ahead: number; behind: number; conflicts: Record | undefined } /** * Refetch the fork status. */ @@ -75,7 +75,11 @@ type EditorStateSchema = { /** * The branch names of current repo. */ - branchNames: Resource + setBranchListEnabled: Setter + /** + * Trigger the branch list to be fetched. + */ + branchList: Resource /** * Additional information about a repository provided by GitHub. */ @@ -503,13 +507,19 @@ export function EditorStateProvider(props: { children: JSXElement }) { } else { setTimeout(() => { const element = document.getElementById("missingTranslation-summary") - element !== null && !filteredMessageLintRules().includes("messageLintRule.inlang.missingTranslation") ? setTourStep("missing-translation-rule") : setTourStep("textfield") + element !== null && + !filteredMessageLintRules().includes("messageLintRule.inlang.missingTranslation") + ? setTourStep("missing-translation-rule") + : setTourStep("textfield") }, 100) } } else if (tourStep() === "missing-translation-rule" && project()) { setTimeout(() => { const element = document.getElementById("missingTranslation-summary") - element !== null && !filteredMessageLintRules().includes("messageLintRule.inlang.missingTranslation") ? setTourStep("missing-translation-rule") : setTourStep("textfield") + element !== null && + !filteredMessageLintRules().includes("messageLintRule.inlang.missingTranslation") + ? setTourStep("missing-translation-rule") + : setTourStep("textfield") }, 100) } }) @@ -541,9 +551,9 @@ export function EditorStateProvider(props: { children: JSXElement }) { ) /** - * createResource is not reacting to changes like: "false","Null", or "undefined". - * Hence, a string needs to be passed to the fetch of the resource. - */ + * createResource is not reacting to changes like: "false","Null", or "undefined". + * Hence, a string needs to be passed to the fetch of the resource. + */ const [userIsCollaborator] = createResource( () => { // do not fetch if no owner or repository is given @@ -596,16 +606,22 @@ export function EditorStateProvider(props: { children: JSXElement }) { } }, async (args) => { + await new Promise((resolve) => setTimeout(resolve, 10000)) + // wait for the browser to be idle + await new Promise((resolve) => requestIdleCallback(resolve)) + + console.info("fetching forkStatus") + const value = await args.repo!.forkStatus() if ("error" in value) { // Silently ignore errors: // The branch might only exist in the fork and not in the upstream repository. - return { ahead: 0, behind: 0, conflicts: false } + return { ahead: 0, behind: 0, conflicts: undefined } } else { return value } }, - { initialValue: { ahead: 0, behind: 0, conflicts: false } } + { initialValue: { ahead: 0, behind: 0, conflicts: undefined } } ) const [previousLoginStatus, setPreviousLoginStatus] = createSignal(localStorage?.user?.isLoggedIn) @@ -641,16 +657,24 @@ export function EditorStateProvider(props: { children: JSXElement }) { } ) - const [branchNames] = createResource( + const [branchListEnabled, setBranchListEnabled] = createSignal(false) + const [branchList] = createResource( () => { + if ( + repo() === undefined || + githubRepositoryInformation() === undefined || + !branchListEnabled() + ) { + return false + } return { repo: repo() } }, async (args) => { + console.info("fetching branchList") return await args.repo?.getBranches() } ) - return ( { ], }) ).toBe(true) -}) \ No newline at end of file +}) diff --git a/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/rewrite.ts b/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/rewrite.ts index 2ec6ab104a..8ad82c241e 100644 --- a/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/rewrite.ts +++ b/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/rewrite.ts @@ -81,7 +81,7 @@ export const rewrite = ({ ? attrubuteValuesToJSValue( langAttribute.value, originalCode - ) + ) : "undefined" } )` diff --git a/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/types.d.ts b/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/types.d.ts index 8641d009e1..8f900e0ed7 100644 --- a/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/types.d.ts +++ b/inlang/source-code/paraglide/paraglide-sveltekit/src/vite/preprocessor/types.d.ts @@ -12,7 +12,7 @@ export type ElementNode = { ? { name: "svelte:element" tag: string | Expression - } + } : { name: Name }) type Expression = { diff --git a/inlang/source-code/sdk/src/createNodeishFsWithWatcher.ts b/inlang/source-code/sdk/src/createNodeishFsWithWatcher.ts index 473676a772..6c7b7a2d8f 100644 --- a/inlang/source-code/sdk/src/createNodeishFsWithWatcher.ts +++ b/inlang/source-code/sdk/src/createNodeishFsWithWatcher.ts @@ -20,7 +20,7 @@ export const createNodeishFsWithWatcher = (args: { ac.abort() } // release references - abortControllers = []; + abortControllers = [] } const makeWatcher = (path: string) => { diff --git a/lix/packages/.dockerignore b/lix/packages/.dockerignore index 8dc0fbfdf6..6627b046db 100644 --- a/lix/packages/.dockerignore +++ b/lix/packages/.dockerignore @@ -1,2 +1,3 @@ node_modules docker-compose.yaml +gitea \ No newline at end of file diff --git a/lix/packages/client/package.json b/lix/packages/client/package.json index 2e92fd21e3..11bf4871f4 100644 --- a/lix/packages/client/package.json +++ b/lix/packages/client/package.json @@ -33,7 +33,7 @@ "async-lock": "^1.4.1", "clean-git-ref": "^2.0.1", "crc-32": "^1.2.2", - "diff3": "^0.0.4", + "diff3": "./vendored/diff3", "ignore": "^5.3.1", "octokit": "3.1.2", "pako": "^1.0.11", diff --git a/lix/packages/client/src/git-http/client.js b/lix/packages/client/src/git-http/client.js deleted file mode 100644 index 28e2928413..0000000000 --- a/lix/packages/client/src/git-http/client.js +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Forked from https://github.com/isomorphic-git/isomorphic-git/blob/main/src/http/web/index.js - * for credentials: "include" support, configurable payload overrides, configurable logging etc. - * @typedef {Object} GitProgressEvent - * @property {string} phase - * @property {number} loaded - * @property {number} total - */ - -/** - * @callback ProgressCallback - * @param {GitProgressEvent} progress - * @returns {void | Promise} - */ - -/** - * @typedef {Object} GitHttpRequest - * @property {string} url - The URL to request - * @property {string} [method='GET'] - The HTTP method to use - * @property {Object} [headers={}] - Headers to include in the HTTP request - * @property {Object} [agent] - An HTTP or HTTPS agent that manages connections for the HTTP client (Node.js only) - * @property {AsyncIterableIterator} [body] - An async iterator of Uint8Arrays that make up the body of POST requests - * @property {ProgressCallback} [onProgress] - Reserved for future use (emitting `GitProgressEvent`s) - * @property {object} [signal] - Reserved for future use (canceling a request) - */ - -/** - * @typedef {Object} GitHttpResponse - * @property {string} url - The final URL that was fetched after any redirects - * @property {string} [method] - The HTTP method that was used - * @property {Object} [headers] - HTTP response headers - * @property {AsyncIterableIterator} [body] - An async iterator of Uint8Arrays that make up the body of the response - * @property {number} statusCode - The HTTP status code - * @property {string} statusMessage - The HTTP status message - */ - -/** - * @callback HttpFetch - * @param {GitHttpRequest} request - * @returns {Promise} - */ - -/** - * @typedef {Object} HttpClient - * @property {HttpFetch} request - */ - -// @ts-nocheck - -// Convert a value to an Async Iterator -// This will be easier with async generator functions. -function fromValue(value) { - let queue = [value] - return { - next() { - return Promise.resolve({ done: queue.length === 0, value: queue.pop() }) - }, - return() { - queue = [] - return {} - }, - [Symbol.asyncIterator]() { - return this - }, - } -} - -function getIterator(iterable) { - if (iterable[Symbol.asyncIterator]) { - return iterable[Symbol.asyncIterator]() - } - if (iterable[Symbol.iterator]) { - return iterable[Symbol.iterator]() - } - if (iterable.next) { - return iterable - } - return fromValue(iterable) -} - -// Currently 'for await' upsets my linters. -async function forAwait(iterable, cb) { - const iter = getIterator(iterable) - // eslint-disable-next-line no-constant-condition - while (true) { - const { value, done } = await iter.next() - if (value) await cb(value) - if (done) break - } - if (iter.return) iter.return() -} - -async function collect(iterable) { - let size = 0 - const buffers = [] - // This will be easier once `for await ... of` loops are available. - await forAwait(iterable, (value) => { - buffers.push(value) - size += value.byteLength - }) - const result = new Uint8Array(size) - let nextIndex = 0 - for (const buffer of buffers) { - result.set(buffer, nextIndex) - nextIndex += buffer.byteLength - } - return result -} - -// Convert a web ReadableStream (not Node stream!) to an Async Iterator -// adapted from https://jakearchibald.com/2017/async-iterators-and-generators/ -function fromStream(stream) { - // Use native async iteration if it's available. - if (stream[Symbol.asyncIterator]) return stream - const reader = stream.getReader() - return { - next() { - return reader.read() - }, - return() { - reader.releaseLock() - return {} - }, - [Symbol.asyncIterator]() { - return this - }, - } -} - -/* eslint-env browser */ - -/** - * MakeHttpClient - * - * @param { verbose?: boolean, desciption?: string, onReq: ({body: any, url: string }) => {body: any, url: string} } - * @returns HttpClient - */ -export function makeHttpClient({ verbose, description, onReq, onRes }) { - /** - * HttpClient - * - * @param {GitHttpRequest} request - * @returns {Promise} - */ - async function request({ url, method = "GET", headers = {}, body }) { - // onProgress param not used - // streaming uploads aren't possible yet in the browser - - if (body) { - body = await collect(body) - } - const origUrl = url - const origMethod = method - - if (onReq) { - const rewritten = await onReq({ body, url }) - - method = rewritten?.method || method - headers = rewritten?.headers || headers - body = rewritten?.body || body - url = rewritten?.url || url - } - - const res = await fetch(url, { method, headers, body, credentials: "include" }) - - // convert Header object to ordinary JSON - let resHeaders = {} - for (const [key, value] of res.headers.entries()) { - resHeaders[key] = value - } - - if (verbose) { - console.warn(`${description} git req:`, origUrl) - } - - const statusCode = res.status - - let resBody - if (onRes) { - const uint8Array = new Uint8Array(await res.arrayBuffer()) - const rewritten = await onRes({ - origUrl, - usedUrl: url, - resBody: uint8Array, - statusCode, - resHeaders, - }) - - resHeaders = rewritten?.resHeaders || resHeaders - resBody = rewritten?.resBody || [uint8Array] - } - - if (!resBody) { - resBody = - res.body && res.body.getReader - ? fromStream(res.body) - : [new Uint8Array(await res.arrayBuffer())] - } - - return { - url: origUrl, - method: origMethod, - statusCode, - statusMessage: res.statusText, - body: resBody, - headers: resHeaders, - } - } - - return { request } -} diff --git a/lix/packages/client/src/git-http/client.ts b/lix/packages/client/src/git-http/client.ts new file mode 100644 index 0000000000..b66864c418 --- /dev/null +++ b/lix/packages/client/src/git-http/client.ts @@ -0,0 +1,245 @@ +/** + * Forked from https://github.com/isomorphic-git/isomorphic-git/blob/main/src/http/web/index.js + * for credentials: "include" support, configurable payload overrides, configurable logging etc. + * @typedef {Object} GitProgressEvent + * @property {string} phase + * @property {number} loaded + * @property {number} total + * @callback ProgressCallback + * @param {GitProgressEvent} progress + * @returns {void | Promise} + */ + +interface GitHttpRequest { + url: string + method?: string + headers?: Record + agent?: object + body?: AsyncIterableIterator + onProgress?: any // Replace 'any' with the actual type ProgressCallback if available + signal?: object +} + +interface GitHttpResponse { + url: string + method?: string + headers?: Record + body?: AsyncIterableIterator + statusCode: number + statusMessage: string +} + +type HttpFetch = (request: GitHttpRequest) => Promise + +interface HttpClient { + request: HttpFetch +} + +// Convert a value to an Async Iterator +// This will be easier with async generator functions. +function fromValue(value: any) { + let queue = [value] + return { + next() { + return Promise.resolve({ done: queue.length === 0, value: queue.pop() }) + }, + return() { + queue = [] + return {} + }, + [Symbol.asyncIterator]() { + return this + }, + } +} + +function getIterator(iterable: any) { + if (iterable[Symbol.asyncIterator]) { + return iterable[Symbol.asyncIterator]() + } + if (iterable[Symbol.iterator]) { + return iterable[Symbol.iterator]() + } + if (iterable.next) { + return iterable + } + return fromValue(iterable) +} + +// Currently 'for await' upsets my linters. +async function forAwait(iterable: any, cb: (arg: any) => void) { + const iter = getIterator(iterable) + // eslint-disable-next-line no-constant-condition + while (true) { + const { value, done } = await iter.next() + if (value) await cb(value) + if (done) break + } + if (iter.return) iter.return() +} + +async function collect(iterable: any) { + let size = 0 + const buffers: any[] = [] + // This will be easier once `for await ... of` loops are available. + await forAwait(iterable, (value: any) => { + buffers.push(value) + size += value.byteLength + }) + const result = new Uint8Array(size) + let nextIndex = 0 + for (const buffer of buffers) { + result.set(buffer, nextIndex) + nextIndex += buffer.byteLength + } + return result +} + +// Convert a web ReadableStream (not Node stream!) to an Async Iterator +// adapted from https://jakearchibald.com/2017/async-iterators-and-generators/ +// function fromStream(stream: any) { +// // Use native async iteration if it's available. +// if (stream[Symbol.asyncIterator]) return stream +// const reader = stream.getReader() +// return { +// next() { +// return reader.read() +// }, +// return() { +// reader.releaseLock() +// return {} +// }, +// [Symbol.asyncIterator]() { +// return this +// }, +// } +// } +type MakeHttpClientArgs = { + debug?: boolean + description?: string + onRes?: ({ + usedUrl, + origUrl, + resBody, + statusCode, + resHeaders, + }: { + usedUrl: string + origUrl: string + resBody: Uint8Array + statusCode: number + resHeaders: Record + }) => any + onReq?: ({ body, url, method }: { body: any; url: string; method: string }) => any +} + +// The init cache is responsible for deduplicating all reqs happening in the first 15 seconds of application start. +// this is to get rid of redundant primer calls for the git protocol. after this we disable caching completely to avoid unexpecting sideffect while using +let cache: Map | undefined = new Map() +let cacheDisabler: any +export function makeHttpClient({ + debug, + description, + onReq, + onRes, +}: MakeHttpClientArgs): HttpClient { + async function request({ + url, + method = "GET", + headers = {}, + body: rawBody, + }: GitHttpRequest): Promise { + // onProgress param not used + // streaming uploads aren't possible yet in the browser + let body = rawBody ? await collect(rawBody) : undefined + + const origUrl = url + const origMethod = method + + if (cache && origMethod === "GET" && cache.has(origUrl)) { + const { resHeaders, resBody } = cache.get(origUrl) + return { + url: origUrl, + method: origMethod, + statusCode: 200, + statusMessage: "OK", + body: resBody, + headers: resHeaders, + } + } + + if (onReq) { + const rewritten = await onReq({ body, url, method }) + + method = rewritten?.method || method + headers = rewritten?.headers || headers + body = rewritten?.body || body + url = rewritten?.url || url + } + + const res = await fetch(url, { method, headers, body, credentials: "include" }) + + // convert Header object to ordinary JSON + let resHeaders: Record = {} + // @ts-ignore -- headers has entries but ts complains + for (const [key, value] of res.headers.entries()) { + resHeaders[key] = value + } + + if (debug) { + console.warn(`${description} git req:`, origUrl) + } + + const statusCode = res.status + + let resBody + + const uint8Array = res.body && new Uint8Array(await res.arrayBuffer()) + + if (debug && uint8Array) { + const { inflatePackResponse } = await import("../git/debug/packfile.js") + console.info(await inflatePackResponse(uint8Array).catch((err: any) => err)) + } + + if (onRes) { + const rewritten = await onRes({ + origUrl, + usedUrl: url, + resBody: uint8Array!, + statusCode, + resHeaders, + }) + + resHeaders = rewritten?.resHeaders || resHeaders + resBody = rewritten?.resBody || [uint8Array] + } + + if (!resBody) { + resBody = [uint8Array] + // @ts-ignore -- done by isogit, not sure why + // TODO: prefer stream over uint8Array? + // res.body && res.body.getReader ? fromStream(res.body) : [uint8Array] + } + + if (cache && statusCode === 200 && origMethod === "GET") { + if (!cacheDisabler) { + cacheDisabler = setTimeout(() => { + cache?.clear() + cache = undefined + }, 15000) + } + cache.set(origUrl, { resHeaders, resBody }) + } + + return { + url: origUrl, + method: origMethod, + statusCode, + statusMessage: res.statusText, + body: resBody, + headers: resHeaders, + } + } + + return { request } +} diff --git a/lix/packages/client/src/git-http/helpers.ts b/lix/packages/client/src/git-http/helpers.ts index 7b11bc4158..7f256b562d 100644 --- a/lix/packages/client/src/git-http/helpers.ts +++ b/lix/packages/client/src/git-http/helpers.ts @@ -1,8 +1,283 @@ +import type { NodeishFilesystem } from "@lix-js/fs" +// we need readObject to use format deflated, possibly move to _readObject later +import { readObject } from "../../vendored/isomorphic-git/index.js" + +export function dirname(path: string) { + const last = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\")) + if (last === -1) return "." + if (last === 0) return "/" + return path.slice(0, last) +} + +export function basename(path: string) { + const last = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\")) + if (last > -1) { + path = path.slice(last + 1) + } + return path +} + function padHex(pad: number, n: number) { const s = n.toString(16) return "0".repeat(pad - s.length) + s } +const WANT_PREFIX = "want " + +/* + * overrides all wants in the given lines with the ids provided in oids + * + * a set of orginal lines used to fetch a commit + * [ + * "want d7e62aef79d771d1771cb44c9e01faa4b7a607fe multi_ack_detailed no-done side-band-64k ofs-delta agent=git/isomorphic-git@1.24.5\n", + * "deepen 1\n", + * "", // flush + * "done\n", + * ] + * with an array of oids passed like + * [ + * "1111111111111111111111111111111111111111", + * "2222222222222222222222222222222222222222", + * ] + * + * would result in: + * [ + * "want 1111111111111111111111111111111111111111 multi_ack_detailed no-done side-band-64k ofs-delta agent=git/isomorphic-git@1.24.5\n", + * "want 2222222222222222222222222222222222222222\n" + * "deepen 1\n", + * "", // flush + * "done\n", + * ] + * + * @param lines the lines of the original request + * @returns the updated lines with wants lines containing the passed oids + */ +export function overrideWants(lines: string[], oids: string[]) { + const newLines = [] + let wantsCount = 0 + + let lastLineWasAWants = false + + // override existing wants + for (const line of lines) { + if (line.startsWith(WANT_PREFIX)) { + lastLineWasAWants = true + if (oids.length > wantsCount) { + const postOidCurrentLine = line.slice( + Math.max(0, WANT_PREFIX.length + oids[wantsCount]!.length) + ) + newLines.push(`${WANT_PREFIX}${oids[wantsCount]}${postOidCurrentLine}`) + } + wantsCount += 1 + } + + if (!line.startsWith(WANT_PREFIX)) { + if (lastLineWasAWants && oids.length > wantsCount) { + while (oids.length > wantsCount) { + newLines.push(WANT_PREFIX + oids[wantsCount] + "\n") + wantsCount += 1 + } + lastLineWasAWants = false + } + newLines.push(line) + } + } + + return newLines +} + +/* + * + * adds "allow-tip-sha1-in-want", "allow-reachable-sha1-in-want" and "no-progress" as capabilities (appends it to the first wants line found) to be able to fetch specific blobs by there hash + * compare: https://git-scm.com/docs/git-rev-list#Documentation/git-rev-list.txt---filterltfilter-specgt + * + * The Orginal lines without the capabilities could look like this: + * + * [ + * "want d7e62aef79d771d1771cb44c9e01faa4b7a607fe multi_ack_detailed no-done side-band-64k ofs-delta agent=git/isomorphic-git@1.24.5\n", + * "deepen 1\n", + * "", // flush + * "done\n", + * ] + * ---- + * + * + * The returned lines will look like this: + * + * [ + * "want d7e62aef79d771d1771cb44c9e01faa4b7a607fe multi_ack_detailed no-done side-band-64k ofs-delta agent=git/isomorphic-git@1.24.5 allow-tip-sha1-in-want allow-reachable-sha1-in-want no-progress\n", + * "deepen 1\n", + * "", // flush + * "done\n", + * ] + * + * @param lines the lines of the original request + * @returns + */ +export function addWantsCapabilities(lines: string[]) { + let capabilitiesAdded = false + const updatedLines = [] + for (let line of lines) { + if (line.startsWith(WANT_PREFIX) && !capabilitiesAdded) { + // lets take the original line withouth the trailing \n and add the new capabilities + // no-progress to skip informal stream that gives input about objects packed etc (https://git-scm.com/docs/protocol-capabilities#_no_progress) + // allow-tip-sha1-in-want allow-reachable-sha1-in-want to use wants https://git-scm.com/docs/protocol-capabilities#_allow_reachable_sha1_in_want // TODO #1459 check what if we can only use allow-reachable-sha1-in-want + line = + line.slice(0, Math.max(0, line.length - 1)) + + " allow-tip-sha1-in-want allow-reachable-sha1-in-want\n" + line = line.replace("ofs-delta", "") + capabilitiesAdded = true + } + updatedLines.push(line) + } + return updatedLines +} +export function addNoProgress(lines: string[]) { + let capabilitiesAdded = false + const updatedLines = [] + for (let line of lines) { + if (line.startsWith(WANT_PREFIX) && !capabilitiesAdded) { + line = line.slice(0, Math.max(0, line.length - 1)) + " no-progress\n" + capabilitiesAdded = true + } + updatedLines.push(line) + } + return updatedLines +} + +/* + * + * adds filter=blob:none to the request represented by the given lines + * compare: https://git-scm.com/docs/git-rev-list#Documentation/git-rev-list.txt---filterltfilter-specgt + * + * The Orginal lines without blob filter could look like this: + * + * [ + * "want d7e62aef79d771d1771cb44c9e01faa4b7a607fe multi_ack_detailed no-done side-band-64k ofs-delta agent=git/isomorphic-git@1.24.5", + * "deepen 1", + * "", + * "done", + * ] + * ---- + * + * The returned lines will add the filter as capability into the first want line and adds filter blob:none after the deepen 1 line + * + * The returned lines will look like this: + * + * [ + * "want d7e62aef79d771d1771cb44c9e01faa4b7a607fe multi_ack_detailed no-done side-band-64k ofs-delta agent=git/isomorphic-git@1.24.5 filter", + * "deepen 1", + * "filter blob:none" + * "", + * "done", + * ] + * + * @param lines the lines of the original request + * @returns + */ +export function addBlobNoneFilter(lines: string[]) { + let filterCapabilityAdded = false + let filterAdded = false + + const updatedLines = [] + const flushLine = "" + + for (let line of lines) { + // finds the first wants line - and append the filter capability and adds "filter" after last wants line - this is capability declaration is needed for filter=blob:none to work + // see: https://git-scm.com/docs/protocol-capabilities#_filter + if (line.startsWith("want") && !filterCapabilityAdded) { + line = line.slice(0, Math.max(0, line.length - 1)) + " filter\n" + filterCapabilityAdded = true + } + + // insert the filter blon:none before the deepen since or the deepen not if both not exist before the flush... + // see: https://git-scm.com/docs/git-rev-list#Documentation/git-rev-list.txt---filterltfilter-specgt + if ( + !filterAdded && + (line.startsWith("deepen-since") || line.startsWith("deepen-not") || line === flushLine) + ) { + updatedLines.push("filter blob:none\n") + filterAdded = true + } + + updatedLines.push(line) + } + + return updatedLines +} + +/** + * Helper method taking lines about to be sent to git-upload-pack and replaceses the haves part with the overrides provided + * NOTE: this method was used to fetch only a subset of oids by building by substracting the them from all oids from the repo + * now that we foud out about "allow-tip-sha1-in-want allow-reachable-sha1-in-want " capabilites we no longer use this + * @param lines + * @param oids + * @returns + */ +export function overrideHaves(lines: string[], oids: string[]) { + // flush line is the only line with an empty string + const linesWithoutHaves = [] + const flushLine = "" + + // delete existing haves + for (const line of lines) { + if (!line.startsWith("have ")) { + linesWithoutHaves.push(line) + } + } + + const updatedLines = [] + for (const line of linesWithoutHaves) { + updatedLines.push(line) + if (line === flushLine) { + for (const oid of oids) { + updatedLines.push("have " + oid + "\n") + } + } + } + return updatedLines +} + +/** + * This helper function checks the git folder for the existance of a blob. May it be within a pack file or as loose object. + * It returns true if the blob can be found and false if not. + * + * @param fs the filesystem that has access to the repository. MUST not be an intercepted filesystem since this would lead to recruision // TODO #1459 we may want to check this at runtime / compiletime + * @param oid the hash of the content of a file or a file we want to check if it exists locally // TODO #1459 think about folders here + * @param gitdir the dire to look for blobs in + * @returns + */ +export async function blobExistsLocaly({ + fs, + oid, + gitdir, +}: { + fs: NodeishFilesystem + oid: string + gitdir: string +}) { + try { + // TODO: if we touch this again, maybe move to just checking pack file index to avoid overhead reading the object + await readObject({ + // fs must not be intercepted - we don't want to intercept calls of read Object // TODO #1459 can we check this by type checking or an added flag property for better dx? + fs, + oid, + gitdir, + // NOTE: we use deflated to stop early in _readObject no hashing etc is happening for format deflated + format: "deflated", + }) + + // read object will fail with a thrown error if it can't find an object with the fiven oid... + return true + } catch (err) { + // we only expect "Error NO ENTry" or iso-gits NotFoundError - rethrow on others + if ((err as any).code !== "ENOENT" && (err as any).code !== "NotFoundError") { + throw err + } + } + return false +} + /** * Takes a line and addes the line lenght as 4 digit fixed hex value at the beginning… * @@ -79,6 +354,8 @@ export function decodeGitPackLines(concatenatedUint8Array: Uint8Array) { strings.push(stringData) } } + // Add the string to the array + // Move the offset to the next potential string return strings } diff --git a/lix/packages/client/src/git-http/optimize-refs.ts b/lix/packages/client/src/git-http/optimize-refs.ts deleted file mode 100644 index 0758f7bf73..0000000000 --- a/lix/packages/client/src/git-http/optimize-refs.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { encodePackLine, decodeGitPackLines } from "./helpers.js" - -export async function optimizedRefsReq({ - url, - addRef, -}: { - url: string - addRef?: string - body?: any -}) { - // "http://localhost:3001/git-proxy//github.com/inlang/example/info/refs?service=git-upload-pack" - if (!url.endsWith("info/refs?service=git-upload-pack")) { - return - } - - // create new url - const uploadPackUrl = url.replace("info/refs?service=git-upload-pack", "git-upload-pack") - // create new body - const lines = [] - - lines.push(encodePackLine("command=ls-refs")) - // 0001 - Delimiter Packet (delim-pkt) - separates sections of a message - lines.push(encodePackLine("agent=git/isomorphic-git@1.24.5") + "0001") - // we prefix refs/heads hardcoded here since the ref is set to main.... - if (addRef) { - lines.push(encodePackLine("ref-prefix refs/heads/" + addRef)) - } - lines.push(encodePackLine("ref-prefix HEAD")) - lines.push(encodePackLine("symrefs")) - lines.push(encodePackLine("")) - - return { - url: uploadPackUrl, - method: "POST", - headers: { - accept: "application/x-git-upload-pack-result", - "content-type": "application/x-git-upload-pack-request", - "git-protocol": "version=2", - }, - // here we expect the body to be a string - we can just use the array and join it - body: lines.join(""), - } -} - -export async function optimizedRefsRes({ - origUrl, - resBody, - statusCode, - resHeaders, -}: { - origUrl: string - usedUrl: string - resBody: Uint8Array - statusCode: number - resHeaders: Record -}) { - if (!origUrl.endsWith("info/refs?service=git-upload-pack")) { - return - } - - if (statusCode !== 200) { - return - } else { - let headSymref = "" - - const origLines = decodeGitPackLines(resBody) - const rewrittenLines = ["# service=git-upload-pack\n", ""] - - const capabilites = - "\x00multi_ack thin-pack side-band side-band-64k ofs-delta shallow deepen-since deepen-not deepen-relative no-progress include-tag multi_ack_detailed allow-tip-sha1-in-want allow-reachable-sha1-in-want no-done filter object-format=sha1" - - for (const line of origLines) { - if (line.includes("HEAD symref-target")) { - // 0050d7e62aef79d771d1771cb44c9e01faa4b7a607fe HEAD symref-target: -> length - headSymref = "refs" + line.slice(64) - headSymref = headSymref.endsWith("\n") ? headSymref.slice(0, -1) : headSymref - const headBlob = line.slice(0, 40) - rewrittenLines.push(headBlob + " HEAD" + capabilites + " symref=HEAD:" + headSymref) - - rewrittenLines.push(headBlob + " " + headSymref) - } else { - rewrittenLines.push(line) - } - } - - rewrittenLines.push("") - - resHeaders["content-type"] = "application/x-git-upload-pack-advertisement" - - const bodyString = rewrittenLines - .map((updatedRawLine) => encodePackLine(updatedRawLine)) - .join("") - - return { - resHeaders, - - resBody: [new TextEncoder().encode(bodyString)], - } - } -} diff --git a/lix/packages/client/src/git-http/optimizeReq.ts b/lix/packages/client/src/git-http/optimizeReq.ts new file mode 100644 index 0000000000..d5eb5c4172 --- /dev/null +++ b/lix/packages/client/src/git-http/optimizeReq.ts @@ -0,0 +1,189 @@ +import { + decodeGitPackLines, + addBlobNoneFilter, + overrideHaves, + addWantsCapabilities, + overrideWants, + encodePackLine, + addNoProgress, +} from "./helpers.js" + +/*** + * This takes the request, decodes the request body and extracts each line in the format of the git-upload-pack protocol (https://git-scm.com/docs/gitprotocol-v2) + * and allows us to rewrite the request to add filters like blob:none or request only specific oids (overrideWants) or block list specific oids (overrideHaves) + */ +export async function optimizeReq( + gitConfig: { + noBlobs?: boolean + + addRefs?: string[] + + overrideHaves?: string[] + overrideWants?: string[] + } = {}, + { + method, + url, + body, + }: { + method: string + url: string + body?: any + } +) { + // Optimize refs requests, GET info/refs?service=git-upload-pack is a request to get the refs but does not support filtering, we rewrite this to a post upload pack v2 request that allows filetering refs how we want + // FIXME: document url + method api, how to get tree for commit just filter file blobs + // "http://localhost:3001/git-proxy//github.com/inlang/example/info/refs?service=git-upload-pack" + if (url.endsWith("info/refs?service=git-upload-pack") && gitConfig.addRefs !== undefined) { + const uploadPackUrl = url.replace("info/refs?service=git-upload-pack", "git-upload-pack") + const lines = [] + + // TODO: #1459 check if we have to ask for the symrefs + lines.push(encodePackLine("command=ls-refs")) + + // 0001 - Delimiter Packet (delim-pkt) - separates sections of a message + lines.push(encodePackLine("agent=lix") + "0001") + + if (gitConfig.addRefs?.length > 0) { + for (let ref of gitConfig.addRefs) { + if (!ref.startsWith("refs/")) { + ref = "refs/heads/" + ref + } + lines.push(encodePackLine("ref-prefix " + ref)) + } + } + + lines.push(encodePackLine("ref-prefix HEAD")) + + // TODO: what does this do? + lines.push(encodePackLine("symrefs")) + // empty line for empty symrefs + lines.push(encodePackLine("")) + + return { + url: uploadPackUrl, + body: lines.join(""), + method: "POST", + headers: { + accept: "application/x-git-upload-pack-result", + "content-type": "application/x-git-upload-pack-request", + "git-protocol": "version=2", + }, + } + } + + // Post requests are always /git-upload-pack requests for the features we currently optimize + // && url.endsWith("/git-upload-pack") + if (method === "POST") { + // decode the lines to be able to change them + let rawLines = decodeGitPackLines(body) + + // We modify the raw lines without encoding, we encode them later at once + if (gitConfig.noBlobs) { + rawLines = addBlobNoneFilter(rawLines) + } + + if (gitConfig.overrideHaves) { + rawLines = overrideHaves(rawLines, gitConfig.overrideHaves) + } + + rawLines = addNoProgress(rawLines) + + // todo add caps helper and always add no progress! + if (gitConfig.overrideWants) { + rawLines = addWantsCapabilities(rawLines) + rawLines = overrideWants(rawLines, gitConfig.overrideWants) + } + + // encode lines again to send them in a reques + const newBody = rawLines.map((updatedRawLine: any) => encodePackLine(updatedRawLine)).join("") + + return { + body: newBody, + } + } + + return undefined +} + +export async function optimizeRes({ + origUrl, + resBody, + statusCode, + resHeaders, +}: { + origUrl: string + usedUrl: string + resBody: Uint8Array + statusCode: number + resHeaders: Record +}) { + // We only optimize ref requests + if (!origUrl.endsWith("info/refs?service=git-upload-pack") || statusCode !== 200) { + return undefined + } + + // TODO: document why we need to override cpabilities + const capabilites = [ + "multi_ack", + "thin-pack", + "side-band", + "side-band-64k", + "ofs-delta", + "shallow", + "deepen-since", + "deepen-not", + "deepen-relative", + "no-progress", + "include-tag", + "multi_ack_detailed", + "allow-tip-sha1-in-want", + "allow-reachable-sha1-in-want", + "no-done", + "filter", + "object-format=sha1", + ] + + const origLines = decodeGitPackLines(resBody) + const rewrittenLines = ["# service=git-upload-pack\n", ""] + + // incoming: + // 0050306e984ae7479b5c7ffc2ef469091e30cfb31393 HEAD symref-target:refs/heads/main + // 00459bb50f447d8c496da3d4a556cc589e5eb2f567e2 refs/heads/test-symlink + // 0000 + + let headSymref = "" + for (const line of origLines) { + if (line.includes("HEAD symref-target")) { + headSymref = "refs" + line.slice(64).replace("\n", "") // /heads/main + + const headBlob = line.slice(0, 40) // '306e984ae7479b5c7ffc2ef469091e30cfb31393' + + rewrittenLines.push( + headBlob + " HEAD" + "\x00" + capabilites.join(" ") + " symref=HEAD:" + headSymref + ) + + rewrittenLines.push(headBlob + " " + headSymref) + } else { + rewrittenLines.push(line.replace("\n", "")) + } + } + + // new line has flush meaning + rewrittenLines.push("") + + // outgoing: + // '# service=git-upload-pack\n', + // '', + // '306e984ae7479b5c7ffc2ef469091e30cfb31393 HEAD\x00 symref=HEAD:refs/heads/main', + // '306e984ae7479b5c7ffc2ef469091e30cfb31393 refs/heads/main', + // '9bb50f447d8c496da3d4a556cc589e5eb2f567e2 refs/heads/test-symlink' + + resHeaders["content-type"] = "application/x-git-upload-pack-advertisement" + const bodyString = rewrittenLines.map((updatedRawLine) => encodePackLine(updatedRawLine)).join("") + + return { + resHeaders, + resBody: [new TextEncoder().encode(bodyString)], + } +} diff --git a/lix/packages/client/src/git/_checkout.js b/lix/packages/client/src/git/__checkout.js similarity index 99% rename from lix/packages/client/src/git/_checkout.js rename to lix/packages/client/src/git/__checkout.js index 21f51b8df5..5e8455c11e 100644 --- a/lix/packages/client/src/git/_checkout.js +++ b/lix/packages/client/src/git/__checkout.js @@ -31,7 +31,7 @@ import { WORKDIR } from "../../vendored/isomorphic-git/index.js" * @returns {Promise} Resolves successfully when filesystem operations are complete * */ -export async function _checkout({ +export async function __checkout({ fs, cache, onProgress, @@ -53,7 +53,10 @@ export async function _checkout({ // TODO: Figure out what to do if both 'ref' and 'remote' are specified, ref already exists, // and is configured to track a different remote. } catch (err) { - if (ref === "HEAD") throw err + if (ref === "HEAD") { + throw err + } + // If `ref` doesn't exist, create a new remote tracking branch // Figure out the commit to checkout const remoteRef = `${remote}/${ref}` diff --git a/lix/packages/client/src/git/add.ts b/lix/packages/client/src/git/add.ts index f7f2c8593f..e905544998 100644 --- a/lix/packages/client/src/git/add.ts +++ b/lix/packages/client/src/git/add.ts @@ -1,7 +1,9 @@ import isoGit from "../../vendored/isomorphic-git/index.js" -import type { RepoContext } from "../repoContext.js" +import type { RepoContext, RepoState } from "../openRepository.js" + +export async function add(ctx: RepoContext, state: RepoState, filepath: string | string[]) { + await state.ensureFirstBatch() -export async function add(ctx: RepoContext, filepath: string | string[]) { return await isoGit.add({ fs: ctx.rawFs, parallel: true, diff --git a/lix/packages/client/src/git/checkout.ts b/lix/packages/client/src/git/checkout.ts index ae29115430..9a32b5ba5b 100644 --- a/lix/packages/client/src/git/checkout.ts +++ b/lix/packages/client/src/git/checkout.ts @@ -1,5 +1,5 @@ import { _FileSystem, _assertParameter, _join } from "../../vendored/isomorphic-git/index.js" -import { _checkout } from "./_checkout.js" +import { __checkout } from "./__checkout.js" import type { RepoContext, RepoState } from "../openRepository.js" export async function checkout(ctx: RepoContext, state: RepoState, { branch }: { branch: string }) { @@ -11,7 +11,7 @@ export async function checkout(ctx: RepoContext, state: RepoState, { branch }: { ) } - return await doCheckout({ + return await _checkout({ fs: ctx.rawFs, cache: ctx.cache, dir: ctx.dir, @@ -72,7 +72,7 @@ export async function checkout(ctx: RepoContext, state: RepoState, { branch }: { * }) * console.log('done') */ -export async function doCheckout({ +export async function _checkout({ fs, onProgress, dir, @@ -93,7 +93,7 @@ export async function doCheckout({ _assertParameter("gitdir", gitdir) const ref = _ref || "HEAD" - return await _checkout({ + return await __checkout({ fs: new _FileSystem(fs), cache, onProgress, diff --git a/lix/packages/client/src/git/commit.ts b/lix/packages/client/src/git/commit.ts index 5512c0d298..36dce8acd5 100644 --- a/lix/packages/client/src/git/commit.ts +++ b/lix/packages/client/src/git/commit.ts @@ -6,6 +6,7 @@ import { commit as originalIsoGitCommit, type TreeEntry, } from "../../vendored/isomorphic-git/index.js" + import { add } from "./add.js" import { remove } from "./remove.js" import { getDirname, getBasename } from "@lix-js/fs" @@ -59,8 +60,8 @@ export async function commit( } } - additions.length && (await add(ctx, additions)) - deletions.length && (await Promise.all(deletions.map((del) => remove(ctx, del)))) + additions.length && (await add(ctx, state, additions)) + deletions.length && (await Promise.all(deletions.map((del) => remove(ctx, state, del)))) } else { // TODO: commit all } @@ -135,7 +136,6 @@ export async function doCommit({ }) } } - return await writeTree({ fs, dir, tree: entries }) } diff --git a/lix/packages/client/src/git/debug/packfile.ts b/lix/packages/client/src/git/debug/packfile.ts new file mode 100644 index 0000000000..8c78cc45fa --- /dev/null +++ b/lix/packages/client/src/git/debug/packfile.ts @@ -0,0 +1,108 @@ +import isoGit, { + _GitCommit, + _GitPackIndex, + _GitTree, + _collect, +} from "../../../vendored/isomorphic-git/index.js" + +if (window) { + // @ts-expect-error + window.isoGit = isoGit +} + +/** + * Use this to intercept your resoponse function to log the pack files interals + * @param Uint8Array the body of a pack Response + * @returns + */ +export async function inflatePackResponse(packResonseBody: Uint8Array) { + // parse pack response in the same way iso git does it in fetch + const bodyResponse = await isoGit._parseUploadPackResponse([packResonseBody]) + + // body response now contains: + // shallows - the commits that do have parents, but not in the shallow repo and therefore grafts are introduced pretending that these commits have no parents.(?) + // https://git-scm.com/docs/shallow + // unshallows - TODO check mechanism here + // @ts-expect-error - TODO how to make Buffer available to TS? + const packfile = Buffer.from(await _collect(bodyResponse.packfile)) + const packfileSha = packfile.slice(-20).toString("hex") + + if (!packfileSha) { + return "" + } + + return { + acks: bodyResponse.acks, + nak: bodyResponse.nak, + shallows: bodyResponse.shallows, + unshallows: bodyResponse.unshallows, + packfilePath: `objects/pack/pack-${packfileSha}.pack`, + ...(await inflatePackfile(packfile)), + } +} + +export async function inflatePackfile(packfile: any) { + // TODO check how to deal with external ref deltas here - do we want to try to get them locally? + const getExternalRefDelta = (oid: string) => console.warn("trying to catch external ref", oid) // readObject({ fs, cache, gitdir, oid }) + + const idx = await _GitPackIndex.fromPack({ + pack: packfile, + getExternalRefDelta, + onProgress: undefined, + }) + + const inflatedPack = {} as any + const trees = {} as any + // @ts-expect-error + for (const hash of idx.hashes) { + const object = await idx.read({ oid: hash }) + const typeKey = object.type + "s" + + if (!inflatedPack[typeKey]) { + inflatedPack[typeKey] = {} as any + } + if (object.type === "tree") { + trees[hash] = new _GitTree(object.object) + } else if (object.type === "commit") { + const commit = new _GitCommit(object.object) + inflatedPack[typeKey][hash] = commit.parse() + } else if (object.type === "blob") { + object.string = object.object.toString() + inflatedPack[typeKey][hash] = object + } else { + inflatedPack[typeKey][hash] = object + } + } + + Object.values(inflatedPack.commits || {}).forEach((commit: any) => { + inflatedPack.trees[commit.tree] = extractTree(trees, commit.tree) + }) + + // add the remaining trees that are not part of commit trees + inflatedPack.trees = { ...inflatedPack.trees, ...trees } + + return inflatedPack +} + +function extractTree(treeEntries: any, treeHash: string) { + const tree = treeEntries[treeHash] + + if (!tree) { + return {} + } + + const extractedTree: Record = {} + tree._entries.forEach((entry: any) => { + if (entry.type === "tree") { + extractedTree[entry.path] = { + children: extractTree(treeEntries, entry.oid), + ...entry, + } + } else { + extractedTree[entry.path] = entry + } + }) + + delete treeEntries[treeHash] + return extractedTree +} diff --git a/lix/packages/client/src/git/diff.js b/lix/packages/client/src/git/diff.js new file mode 100644 index 0000000000..e759a3bbfc --- /dev/null +++ b/lix/packages/client/src/git/diff.js @@ -0,0 +1,45 @@ +import isoGit from "../../vendored/isomorphic-git/index.js" + +// @ts-ignore +export async function getFileStateChanges(ctx, commitA, commitB) { + return isoGit.walk({ + fs: ctx.rawFs, + dir: ctx.dir, + trees: [isoGit.TREE({ ref: commitA }), isoGit.TREE({ ref: commitB })], + map: async function (filepath, [A, B]) { + // ignore directories + if (filepath === ".") { + return + } + if ((await A?.type()) === "tree" || (await B?.type()) === "tree") { + return + } + + // generate ids + const Aoid = await A?.oid() + const Boid = await B?.oid() + + // determine modification type + let type = "equal" + if (Aoid !== Boid) { + type = "modify" + } + if (Aoid === undefined) { + type = "add" + } + if (Boid === undefined) { + type = "remove" + } + if (Aoid === undefined && Boid === undefined) { + console.log("Something weird happened:") + console.log(A) + console.log(B) + } + + return { + path: `/${filepath}`, + type: type, + } + }, + }) +} diff --git a/lix/packages/client/src/git/getBranches.ts b/lix/packages/client/src/git/getBranches.ts index 9e2c712cb1..c8d6023ea1 100644 --- a/lix/packages/client/src/git/getBranches.ts +++ b/lix/packages/client/src/git/getBranches.ts @@ -13,7 +13,7 @@ export async function getBranches(ctx: RepoContext) { url: ctx.gitUrl, corsProxy: ctx.gitProxyUrl, prefix: "refs/heads", - http: makeHttpClient({ verbose: ctx.debug, description: "getBranches" }), + http: makeHttpClient({ debug: ctx.debug, description: "getBranches" }), }) } catch (_error) { return undefined diff --git a/lix/packages/client/src/git/getCurrentBranch.ts b/lix/packages/client/src/git/getCurrentBranch.ts index c19a0fb0b6..25721be753 100644 --- a/lix/packages/client/src/git/getCurrentBranch.ts +++ b/lix/packages/client/src/git/getCurrentBranch.ts @@ -2,7 +2,7 @@ import type { RepoState, RepoContext } from "../openRepository.js" import isoGit from "../../vendored/isomorphic-git/index.js" export async function getCurrentBranch(ctx: RepoContext, state: RepoState) { - // TODO: maybe make stateless? + // TODO: maybe make stateless, deprecate move to currentRef, baseBranch etc. to support branchless work modes return ( (await isoGit.currentBranch({ fs: state.nodeishFs, diff --git a/lix/packages/client/src/git/log.ts b/lix/packages/client/src/git/log.ts index eba17760cf..3b83280fe6 100644 --- a/lix/packages/client/src/git/log.ts +++ b/lix/packages/client/src/git/log.ts @@ -3,15 +3,24 @@ import type { RepoContext } from "../openRepository.js" export async function log( ctx: RepoContext, - cmdArgs: { depth: number; filepath?: string; ref?: string; since?: Date } + cmdArgs: { + depth: number + filepath?: string + ref?: string + since?: Date + force?: boolean + follow?: boolean + } ) { return await isoGit.log({ fs: ctx.rawFs, - depth: cmdArgs.depth, - filepath: cmdArgs.filepath, + depth: cmdArgs?.depth, + filepath: cmdArgs?.filepath, dir: ctx.dir, - ref: cmdArgs.ref, + ref: cmdArgs?.ref, cache: ctx.cache, - since: cmdArgs.since, + since: cmdArgs?.since, + force: cmdArgs?.force, + follow: cmdArgs?.follow, }) } diff --git a/lix/packages/client/src/git/pull.ts b/lix/packages/client/src/git/pull.ts index 483967e5d7..c3909189c8 100644 --- a/lix/packages/client/src/git/pull.ts +++ b/lix/packages/client/src/git/pull.ts @@ -1,26 +1,49 @@ import isoGit from "../../vendored/isomorphic-git/index.js" import type { RepoContext, RepoState, Author } from "../openRepository.js" import { makeHttpClient } from "../git-http/client.js" -import { doCheckout } from "./checkout.js" +import { _checkout } from "./checkout.js" import { emptyWorkdir } from "../lix/emptyWorkdir.js" +import { optimizeReq, optimizeRes } from "../git-http/optimizeReq.js" +import { checkOutPlaceholders } from "../lix/checkoutPlaceholders.js" +// TODO: i consider pull now bad practice nad deprecated. replace with more specific commands for syncing and updating local state export async function pull( ctx: RepoContext, state: RepoState, - cmdArgs: { singleBranch?: boolean; fastForward?: boolean; author?: Author } = {} + cmdArgs: { singleBranch?: boolean; fastForward?: boolean; author?: Author } ) { if (!ctx.gitUrl) { throw new Error("Could not find repo url, only github supported for pull at the moment") } - const pullFs = state.nodeishFs + + const branchName = + state.branchName || (await isoGit.currentBranch({ fs: ctx.rawFs, dir: "/" })) || "HEAD" + + const oid = await isoGit.resolveRef({ + fs: ctx.rawFs, + dir: "/", + ref: "refs/remotes/origin/" + branchName, + }) + const { commit } = await isoGit.readCommit({ fs: ctx.rawFs, dir: "/", oid }) + const since = new Date(commit.committer.timestamp * 1000) const { fetchHead, fetchHeadDescription } = await isoGit.fetch({ - depth: 5, // TODO: how to handle depth with upstream? reuse logic from fork sync - fs: pullFs, + since, + fs: ctx.rawFs, cache: ctx.cache, - http: makeHttpClient({ verbose: ctx.debug, description: "pull" }), + http: makeHttpClient({ + debug: ctx.debug, + description: "pull", + onReq: ctx.experimentalFeatures.lazyClone + ? optimizeReq.bind(null, { + noBlobs: true, + addRefs: [branchName], + }) + : undefined, + onRes: ctx.experimentalFeatures.lazyClone ? optimizeRes : undefined, + }), corsProxy: ctx.gitProxyUrl, - ref: state.branchName, + ref: branchName, tags: false, dir: ctx.dir, url: ctx.gitUrl, @@ -33,47 +56,72 @@ export async function pull( throw new Error("could not fetch head") } - await isoGit.merge({ - fs: pullFs, - cache: ctx.cache, - dir: ctx.dir, - ours: state.branchName, - theirs: fetchHead, - fastForward: cmdArgs.fastForward, - message: `Merge ${fetchHeadDescription}`, - author: cmdArgs.author || ctx.author, - dryRun: false, - noUpdateBranch: false, - // committer, - // signingKey, - // fastForwardOnly, - }) + let materialized: string[] = [] + // FIXME: there is still a race condition somewhere that can lead to materialized files being deleted if (ctx.experimentalFeatures.lazyClone) { - console.warn( - "enableExperimentalFeatures.lazyClone is set for this repo but pull not fully implemented. disabling lazy files" - ) - - await emptyWorkdir(ctx, state) + materialized = await emptyWorkdir(ctx, state) + ctx.debug && console.info("experimental checkout after pull preload:", materialized) + state.checkedOut.clear() + } - // remember we are now leaving lazy mode - ctx.experimentalFeatures.lazyClone = false + /** + * @typedef {Object} MergeDriverParams + * @property {Array} branches + * @property {Array} contents + * @property {string} path + */ + /** + * @callback MergeDriverCallback + * @param {MergeDriverParams} args + * @return {{cleanMerge: boolean, mergedText: string} | Promise<{cleanMerge: boolean, mergedText: string}>} + */ + const mergeDriver = ({ + branches, + contents, + path, + }: { + branches: string[] + contents: string[] + path: string + }) => { + console.log("mergeDriver", branches, contents, path) + ctx.rawFs.writeFile(path + `.${branches[2]!.slice(0, 4)}.conflict`, contents[2] || "") - ctx.debug && console.info('checking out "HEAD" after pull') + return { cleanMerge: true, mergedText: contents[1] || "" } + } - await doCheckout({ - fs: ctx.rawFs, + const mergeRes = await isoGit + .merge({ + fs: state.nodeishFs, cache: ctx.cache, dir: ctx.dir, - ref: state.branchName, - noCheckout: false, + ours: branchName, + theirs: fetchHead, + fastForward: cmdArgs.fastForward === false ? false : true, + message: `Merge ${fetchHeadDescription}`, + author: cmdArgs.author || ctx.author, + dryRun: false, + noUpdateBranch: false, + abortOnConflict: true, + mergeDriver, + // committer, + // signingKey, + // fastForwardOnly, }) + .catch((error) => ({ error })) + + // @ts-ignore + console.info("mergeRes", { data: mergeRes.data, code: mergeRes.code, error: mergeRes.error }) + + if (ctx.experimentalFeatures.lazyClone) { + await checkOutPlaceholders(ctx, state, { preload: materialized }) } else { - await doCheckout({ + await _checkout({ fs: ctx.rawFs, cache: ctx.cache, dir: ctx.dir, - ref: state.branchName, + ref: branchName, noCheckout: false, }) } diff --git a/lix/packages/client/src/git/push.ts b/lix/packages/client/src/git/push.ts index 4de84865c9..09ac2fd703 100644 --- a/lix/packages/client/src/git/push.ts +++ b/lix/packages/client/src/git/push.ts @@ -7,7 +7,7 @@ export const push = async (ctx: RepoContext, state: RepoState) => { throw new Error("Could not find repo url, only github supported for push at the moment") } return await isoGit.push({ - fs: state.nodeishFs, + fs: ctx.rawFs, url: ctx.gitUrl, cache: ctx.cache, corsProxy: ctx.gitProxyUrl, diff --git a/lix/packages/client/src/git/remove.ts b/lix/packages/client/src/git/remove.ts index f9024871f6..73aaa2b868 100644 --- a/lix/packages/client/src/git/remove.ts +++ b/lix/packages/client/src/git/remove.ts @@ -1,7 +1,9 @@ import isoGit from "../../vendored/isomorphic-git/index.js" -import type { RepoContext } from "../repoContext.js" +import type { RepoContext, RepoState } from "../openRepository.js" + +export async function remove(ctx: RepoContext, state: RepoState, filepath: string) { + await state.ensureFirstBatch() -export async function remove(ctx: RepoContext, filepath: string) { return await isoGit.remove({ fs: ctx.rawFs, dir: ctx.dir, diff --git a/lix/packages/client/src/git/status-list.test.ts b/lix/packages/client/src/git/status-list.test.ts index 49513775e5..f51d39de26 100644 --- a/lix/packages/client/src/git/status-list.test.ts +++ b/lix/packages/client/src/git/status-list.test.ts @@ -44,12 +44,12 @@ describe( ) ).toStrictEqual([ [ - ".env", - "ignored", + ".gitignore", + "unmodified", { - headOid: undefined, - stageOid: undefined, - workdirOid: "ignored", + headOid: "6635cf5542756197081eedaa1ec3a7c2c5a0b537", + stageOid: "6635cf5542756197081eedaa1ec3a7c2c5a0b537", + workdirOid: "6635cf5542756197081eedaa1ec3a7c2c5a0b537", }, ], [ @@ -62,12 +62,12 @@ describe( }, ], [ - ".gitignore", - "unmodified", + ".env", + "ignored", { - headOid: "6635cf5542756197081eedaa1ec3a7c2c5a0b537", - stageOid: "6635cf5542756197081eedaa1ec3a7c2c5a0b537", - workdirOid: "6635cf5542756197081eedaa1ec3a7c2c5a0b537", + headOid: undefined, + stageOid: undefined, + workdirOid: "ignored", }, ], ]) @@ -168,15 +168,6 @@ describe( workdirOid: "42", }, ], - [ - ".git", - "ignored", - { - headOid: undefined, - stageOid: undefined, - workdirOid: "ignored", - }, - ], [ ".gitignore", "*modified", @@ -186,6 +177,15 @@ describe( workdirOid: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", }, ], + [ + ".git", + "ignored", + { + headOid: undefined, + stageOid: undefined, + workdirOid: "ignored", + }, + ], ]) const statusResults = await repository.statusList({ includeStatus: ["unmodified"] }) @@ -457,6 +457,21 @@ describe( if (content.placeholder && typeof snapB.fsMap[path] === "string") { snapB.fsMap[path] = { placeholder: true } } + // delete the known extra packfiles in lazy snapshot + if (path.startsWith("/.git/objects/pack/")) { + for (const packFile of [ + "/.git/objects/pack/pack-02be53df793edd50f0fc133384db246495491658", + "/.git/objects/pack/pack-18fd350c176426a096636f821abddb9998d04d0b", + "/.git/objects/pack/pack-19f977d5eaefc2553ad17e1faaf1bdf35feebf0d", + "/.git/objects/pack/pack-8323a3d068cb7bf606cccc1574ef2be32391a2bc", + "/.git/objects/pack/pack-8c4f18c048db5e7b7075c263a73c808c2bc95336", + "/.git/objects/pack/pack-a51df86e4b8a006e9131a6f053a0e07adc0ea487", + ]) { + if (path.startsWith(packFile)) { + delete snapA.fsMap[path] + } + } + } } for (const [, stat] of Object.entries(snapB.fsStats)) { // @ts-ignore @@ -471,6 +486,8 @@ describe( } delete snapA.fsMap["/.git/index/"] + delete snapA.fsMap["/.git/objects/pack/"] + delete snapB.fsMap["/.git/objects/pack/"] delete snapB.fsMap["/.git/index/"] expect(snapA.fsMap).toStrictEqual(snapB.fsMap) @@ -478,5 +495,5 @@ describe( vi.useRealTimers() }) }, - { timeout: 5000 } + { timeout: 7000 } ) diff --git a/lix/packages/client/src/git/status-list.ts b/lix/packages/client/src/git/status-list.ts index e127afd616..e6d69c8004 100644 --- a/lix/packages/client/src/git/status-list.ts +++ b/lix/packages/client/src/git/status-list.ts @@ -45,6 +45,7 @@ type StatusText = | "*undeleted" // fallback for permutations without existing isogit named status | "unknown" + | "ignored" type StatusList = [string, StatusText][] @@ -65,6 +66,7 @@ function join(...parts: string[]) { } export type StatusArgs = { + ensureFirstBatch: () => Promise fs: NodeishFilesystem /** The [working tree](dir-vs-gitdir.md) directory path */ dir: string @@ -107,6 +109,7 @@ export async function statusList( ): ReturnType { return await _statusList({ fs: ctx.rawFs, + ensureFirstBatch: state.ensureFirstBatch, dir: ctx.dir, cache: ctx.cache, sparseFilter: state.sparseFilter, @@ -121,6 +124,7 @@ export async function statusList( */ export async function _statusList({ fs, + ensureFirstBatch, dir = "/", gitdir = join(dir, ".git"), ref = "HEAD", @@ -132,7 +136,11 @@ export async function _statusList({ addHashes = false, }: StatusArgs): Promise { try { - return await walk({ + // this call will materialzie all gitignore files that are queued on lazy cloning + await ensureFirstBatch() + + const ignoredRes: any[] = [] + const walkRes = await walk({ fs, cache, dir, @@ -153,11 +161,12 @@ export async function _statusList({ if (ignored) { // "ignored" file ignored by a .gitignore rule, will not be shown unless explicitly asked for if (includeStatus.includes("ignored") || filepaths.includes(filepath)) { - return [ + // we have to add ignored results but need to stop iterating into ignored folders... + ignoredRes.push([ filepath, "ignored", { headOid: undefined, workdirOid: "ignored", stageOid: undefined }, - ] + ]) } // eslint-disable-next-line unicorn/no-null -- return null to skip walking of ignored trees (folders) - compare (1) @@ -391,6 +400,8 @@ export async function _statusList({ return [filepath, "unknown", entry] }, }) + + return [...walkRes, ...ignoredRes] } catch (err) { // @ts-ignore err.caller = "lix.status" diff --git a/lix/packages/client/src/github/forkStatus.ts b/lix/packages/client/src/github/forkStatus.ts index ae29f0de86..ae82351f71 100644 --- a/lix/packages/client/src/github/forkStatus.ts +++ b/lix/packages/client/src/github/forkStatus.ts @@ -1,9 +1,10 @@ import isoGit from "../../vendored/isomorphic-git/index.js" import { makeHttpClient } from "../git-http/client.js" -import type { RepoContext, RepoState } from "../openRepository.js" +import type { RepoContext } from "../openRepository.js" +import { optimizeReq, optimizeRes } from "../git-http/optimizeReq.js" import { getMeta } from "../github/getMeta.js" -export async function forkStatus(ctx: RepoContext, state: RepoState) { +export async function forkStatus(ctx: RepoContext) { const { gitUrl, debug, dir, cache, owner, repoName, githubClient, gitProxyUrl } = ctx if (!gitUrl) { @@ -24,10 +25,8 @@ export async function forkStatus(ctx: RepoContext, state: RepoState) { return { error: "repo is not a fork" } } - const forkFs = state.nodeishFs - const useBranchName = await isoGit.currentBranch({ - fs: forkFs, + fs: ctx.rawFs, dir, fullname: false, }) @@ -40,7 +39,7 @@ export async function forkStatus(ctx: RepoContext, state: RepoState) { dir, remote: "upstream", url: "https://" + parent.url, - fs: forkFs, + fs: ctx.rawFs, }) try { @@ -51,27 +50,38 @@ export async function forkStatus(ctx: RepoContext, state: RepoState) { cache, ref: useBranchName, remote: "upstream", - http: makeHttpClient({ debug, description: "forkStatus" }), - fs: forkFs, + http: makeHttpClient({ + debug, + description: "forkStatus", + onReq: ctx.experimentalFeatures.lazyClone + ? optimizeReq.bind(null, { + noBlobs: true, + addRefs: [useBranchName || "HEAD"], + }) + : undefined, + onRes: ctx.experimentalFeatures.lazyClone ? optimizeRes : undefined, + }), + tags: false, + fs: ctx.rawFs, }) } catch (err) { return { error: err } } const currentUpstreamCommit = await isoGit.resolveRef({ - fs: forkFs, + fs: ctx.rawFs, dir: "/", ref: "upstream/" + useBranchName, }) const currentOriginCommit = await isoGit.resolveRef({ - fs: forkFs, + fs: ctx.rawFs, dir: "/", ref: useBranchName, }) if (currentUpstreamCommit === currentOriginCommit) { - return { ahead: 0, behind: 0, conflicts: false } + return { ahead: 0, behind: 0, conflicts: undefined } } const compare = await githubClient @@ -89,35 +99,38 @@ export async function forkStatus(ctx: RepoContext, state: RepoState) { return { error: compare.error || "could not diff repos on github" } } + const ahead: number = compare.data.ahead_by + const behind: number = compare.data.behind_by + // fetch from forks upstream await isoGit.fetch({ - depth: compare.data.behind_by + 1, + depth: behind + 1, remote: "upstream", - cache: cache, + cache, singleBranch: true, dir, ref: useBranchName, http: makeHttpClient({ debug, description: "forkStatus" }), - fs: forkFs, + fs: ctx.rawFs, }) // fetch from fors remote await isoGit.fetch({ - depth: compare.data.ahead_by + 1, - cache: cache, + depth: ahead + 1, + cache, singleBranch: true, ref: useBranchName, dir, http: makeHttpClient({ debug, description: "forkStatus" }), corsProxy: gitProxyUrl, - fs: forkFs, + fs: ctx.rawFs, }) // finally try to merge the changes from upstream - let conflicts = false + let conflicts: { data: string[]; code: string } | undefined try { await isoGit.merge({ - fs: forkFs, + fs: ctx.rawFs, cache, author: { name: "lix" }, dir, @@ -127,8 +140,12 @@ export async function forkStatus(ctx: RepoContext, state: RepoState) { noUpdateBranch: true, abortOnConflict: true, }) - } catch (err) { - conflicts = true + } catch (err: any) { + conflicts = { + data: err.data, + code: err.code, + } + console.warn(conflicts) } - return { ahead: compare.data.ahead_by, behind: compare.data.behind_by, conflicts } + return { ahead, behind, conflicts } } diff --git a/lix/packages/client/src/lix/checkoutPlaceholders.ts b/lix/packages/client/src/lix/checkoutPlaceholders.ts index 68582cb555..b0304782bd 100644 --- a/lix/packages/client/src/lix/checkoutPlaceholders.ts +++ b/lix/packages/client/src/lix/checkoutPlaceholders.ts @@ -1,14 +1,24 @@ import type { RepoState, RepoContext } from "../openRepository.js" import isoGit from "../../vendored/isomorphic-git/index.js" -import { doCheckout } from "../git/checkout.js" +import { _checkout } from "../git/checkout.js" import { modeToFileType } from "../git/helpers.js" -export async function checkOutPlaceholders(ctx: RepoContext, state: RepoState) { +export async function checkOutPlaceholders( + ctx: RepoContext, + state: RepoState, + { + materializeGitignores = true, + preload = [], + }: { + materializeGitignores?: boolean + preload?: string[] + } = {} +) { const { rawFs, cache, dir } = ctx const { branchName, checkedOut, sparseFilter } = state - await doCheckout({ - fs: rawFs, + await _checkout({ + fs: ctx.rawFs, // state.nodeishFs, cache, dir, ref: branchName, @@ -18,6 +28,7 @@ export async function checkOutPlaceholders(ctx: RepoContext, state: RepoState) { const fs = rawFs const gitignoreFiles: string[] = [] + let rootHash: string | undefined await isoGit.walk({ fs, dir, @@ -29,15 +40,20 @@ export async function checkOutPlaceholders(ctx: RepoContext, state: RepoState) { if (!commit) { return undefined } - const fileMode = await commit.mode() - - const fileType = modeToFileType(fileMode) if (fullpath.endsWith(".gitignore")) { gitignoreFiles.push(fullpath) - return undefined } + const fileMode = await commit.mode() + const oid = await commit.oid() + + if (fullpath === ".") { + rootHash = oid + } + + const fileType = modeToFileType(fileMode) + if ( sparseFilter && !sparseFilter({ @@ -53,12 +69,12 @@ export async function checkOutPlaceholders(ctx: RepoContext, state: RepoState) { } if (fileType === "file" && !checkedOut.has(fullpath)) { - await fs._createPlaceholder(fullpath, { mode: fileMode }) + await fs._createPlaceholder(fullpath, { mode: fileMode, oid, rootHash }) return fullpath } if (fileType === "symlink" && !checkedOut.has(fullpath)) { - await fs._createPlaceholder(fullpath, { mode: fileMode }) + await fs._createPlaceholder(fullpath, { mode: fileMode, oid, rootHash }) return fullpath } @@ -67,16 +83,12 @@ export async function checkOutPlaceholders(ctx: RepoContext, state: RepoState) { }, }) - if (gitignoreFiles.length) { - await doCheckout({ - fs: rawFs, - dir, - cache, - ref: branchName, - filepaths: gitignoreFiles, - }) - gitignoreFiles.map((file) => checkedOut.add(file)) + if (gitignoreFiles.length && materializeGitignores) { + // This is only used for testing when opeinging a snapshot and using emptyWorkdir, using lazyFs will hang forewer but we allready have the ignore files in object store + preload = [...gitignoreFiles, ...preload] } - state.pending && (await state.pending) + await state.ensureFirstBatch({ preload }) + + return { gitignoreFiles } } diff --git a/lix/packages/client/src/lix/emptyWorkdir.ts b/lix/packages/client/src/lix/emptyWorkdir.ts index cad0eea2d5..d529730e1d 100644 --- a/lix/packages/client/src/lix/emptyWorkdir.ts +++ b/lix/packages/client/src/lix/emptyWorkdir.ts @@ -3,40 +3,74 @@ import { statusList } from "../git/status-list.js" import type { RepoState, RepoContext } from "../openRepository.js" export async function emptyWorkdir(ctx: RepoContext, state: RepoState) { - const { rawFs, cache, dir } = ctx - const { checkedOut } = state + const { rawFs, cache } = ctx - const statusResult = await statusList(ctx, state) - if (statusResult.length > 0) { + state.pending && (await state.pending) + + const statusResult = await statusList(ctx, state, { includeStatus: ["materialized", "ignored"] }) + + const ignored: string[] = [] + const materialized: string[] = [] + const dirty: string[] = [] + for (const [path, status] of statusResult) { + if (status === "unmodified") { + materialized.push(path) + } else if (status === "ignored") { + ignored.push("/" + path) + } else { + dirty.push(path) + } + } + + if (dirty.length > 0) { + console.error(dirty) throw new Error("could not empty the workdir, uncommitted changes") } - const listing = (await rawFs.readdir("/")).filter((entry) => { - return !checkedOut.has(entry) && entry !== ".git" - }) + const listing = await allFiles(rawFs, ignored) - const notIgnored = ( - await Promise.all( - listing.map((entry) => - isoGit.isIgnored({ fs: rawFs, dir, filepath: entry }).then((ignored) => { - return { ignored, entry } + await Promise.all( + listing.map((entry) => + rawFs + .rm(entry) + .catch((err) => { + console.warn(err) }) - ) + .then(() => + isoGit.remove({ + fs: rawFs, + dir: "/", + cache, + filepath: entry, + }) + ) ) ) - .filter(({ ignored }) => !ignored) - .map(({ entry }) => entry) - - for (const toDelete of notIgnored) { - await rawFs.rm(toDelete, { recursive: true }).catch(() => {}) - - // remove it from isoGit's index as well - await isoGit.remove({ - fs: rawFs, - // ref: args.branch, - dir: "/", - cache, - filepath: toDelete, - }) - } + + return materialized +} + +async function allFiles(fs: any, ignored: string[], root = "/"): Promise { + const entries = await fs.readdir(root) + + const notIgnored = entries.filter((entry: string) => !ignored.includes(root + entry)) + + const withMeta = await Promise.all( + notIgnored.map(async (entry: string) => ({ + name: entry, + isDir: ( + await fs.lstat(root + entry).catch((err: any) => { + console.log(err) + }) + )?.isDirectory?.(), + })) + ) + + const withChildren = await Promise.all( + withMeta.map(async ({ name, isDir }) => + isDir ? allFiles(fs, ignored, root + name + "/") : root + name + ) + ) + + return withChildren.flat().map((entry: string) => entry.replace(/^\//, "")) } diff --git a/lix/packages/client/src/lix/getFirstCommitHash.ts b/lix/packages/client/src/lix/getFirstCommitHash.ts index fd6b722f49..cce9e16eef 100644 --- a/lix/packages/client/src/lix/getFirstCommitHash.ts +++ b/lix/packages/client/src/lix/getFirstCommitHash.ts @@ -20,7 +20,7 @@ export async function getFirstCommitHash(ctx: RepoContext) { singleBranch: true, dir: ctx.dir, depth: 2147483647, // the magic number for all commits - http: makeHttpClient({ verbose: ctx.debug, description: "getFirstCommitHash" }), + http: makeHttpClient({ debug: ctx.debug, description: "getFirstCommitHash" }), corsProxy: ctx.gitProxyUrl, fs: getFirstCommitFs, }) diff --git a/lix/packages/client/src/openRepository.test.ts b/lix/packages/client/src/openRepository.test.ts index f65036d824..4d83f7e398 100644 --- a/lix/packages/client/src/openRepository.test.ts +++ b/lix/packages/client/src/openRepository.test.ts @@ -47,7 +47,6 @@ describe("main workflow", () => { expect(repoUrl).toBe("file:///") - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- test fails if repoUrl is null const repository: Awaited> = await openRepository(repoUrl!, { nodeishFs: fs, branch: "test-symlink", @@ -74,7 +73,6 @@ describe("main workflow", () => { expect(repoUrl).toBe("file:///test/toast") - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- test fails if repoUrl is null const repository: Awaited> = await openRepository(repoUrl!, { nodeishFs: fs, branch: "test-symlink", @@ -119,7 +117,7 @@ describe("main workflow", () => { }) it("can open repo with lazy clone/checkout", async () => { - const lazyRepo = await openRepository("https://github.com/inlang/ci-test-repo", { + const lazyRepo = await openRepository("https://github.com/opral/ci-test-repo", { branch: "test-symlink", nodeishFs: createNodeishMemoryFs(), experimentalFeatures: { lazyClone: true, lixCommit: true }, @@ -323,7 +321,6 @@ describe("main workflow", () => { expect(repoUrl).toBe("file:///") - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- test fails if repoUrl is null const repository: Awaited> = await openRepository(repoUrl!, { nodeishFs: fs, branch: "test-symlink", @@ -358,6 +355,7 @@ describe("main workflow", () => { await repo.nodeishFs.rm("/static/test1") await repo._emptyWorkdir() + await repo._checkOutPlaceholders() await repo.nodeishFs.mkdir("/static/test/nested/deep/deeper", { recursive: true }) @@ -373,15 +371,6 @@ describe("main workflow", () => { ) expect(status).toStrictEqual([ - [ - ".git", - "ignored", - { - headOid: undefined, - stageOid: undefined, - workdirOid: "ignored", - }, - ], [ ".gitignore", "unmodified", @@ -401,25 +390,25 @@ describe("main workflow", () => { }, ], [ - "static/test/nested/deep", - "ignored", + "static/test/nested/test2", + "*untracked", { headOid: undefined, stageOid: undefined, - workdirOid: "ignored", + workdirOid: "42", }, ], [ - "static/test/nested/deep/deeper", - "ignored", + "static/test/testparent", + "*untracked", { headOid: undefined, stageOid: undefined, - workdirOid: "ignored", + workdirOid: "42", }, ], [ - "static/test/nested/deep/deeper/test3", + ".git", "ignored", { headOid: undefined, @@ -428,7 +417,7 @@ describe("main workflow", () => { }, ], [ - "static/test/nested/deep/test1", + "static/test/nested/deep", "ignored", { headOid: undefined, @@ -436,24 +425,6 @@ describe("main workflow", () => { workdirOid: "ignored", }, ], - [ - "static/test/nested/test2", - "*untracked", - { - headOid: undefined, - stageOid: undefined, - workdirOid: "42", - }, - ], - [ - "static/test/testparent", - "*untracked", - { - headOid: undefined, - stageOid: undefined, - workdirOid: "42", - }, - ], ]) }) diff --git a/lix/packages/client/src/openRepository.ts b/lix/packages/client/src/openRepository.ts index 8ef5de2108..d61df9f747 100644 --- a/lix/packages/client/src/openRepository.ts +++ b/lix/packages/client/src/openRepository.ts @@ -81,7 +81,7 @@ export async function openRepository( commit: commit.bind(undefined, ctx, state), status: status.bind(undefined, ctx, state), statusList: statusList.bind(undefined, ctx, state), - forkStatus: forkStatus.bind(undefined, ctx, state), + forkStatus: forkStatus.bind(undefined, ctx), getMeta: getMeta.bind(undefined, ctx), listRemotes: listRemotes.bind(undefined, ctx, state), log: log.bind(undefined, ctx), @@ -100,8 +100,8 @@ export async function openRepository( // only exposed for testing _emptyWorkdir: emptyWorkdir.bind(undefined, ctx, state), _checkOutPlaceholders: checkOutPlaceholders.bind(undefined, ctx, state), - _add: add.bind(undefined, ctx), - _remove: remove.bind(undefined, ctx), + _add: add.bind(undefined, ctx, state), + _remove: remove.bind(undefined, ctx, state), _isoCommit: isoCommit.bind(undefined, ctx), } } diff --git a/lix/packages/client/src/repoContext.ts b/lix/packages/client/src/repoContext.ts index 4bba3d9caf..3a867848bd 100644 --- a/lix/packages/client/src/repoContext.ts +++ b/lix/packages/client/src/repoContext.ts @@ -98,7 +98,7 @@ export async function repoContext( args.experimentalFeatures || (isWhitelistedRepo ? { lazyClone: freshClone, lixCommit: true } : {}) - const useLazyFS = experimentalFeatures?.lazyClone && args.nodeishFs?._createPlaceholder + const useLazyFS = experimentalFeatures?.lazyClone && rawFs?._createPlaceholder const cache = useLazyFS ? {} : undefined diff --git a/lix/packages/client/src/repoState.ts b/lix/packages/client/src/repoState.ts index 222778dc85..f39ce789c7 100644 --- a/lix/packages/client/src/repoState.ts +++ b/lix/packages/client/src/repoState.ts @@ -1,14 +1,12 @@ import { withProxy } from "./helpers.js" import { makeHttpClient } from "./git-http/client.js" -import { optimizedRefsRes, optimizedRefsReq } from "./git-http/optimize-refs.js" -import { doCheckout as lixCheckout } from "./git/checkout.js" +import { optimizeReq, optimizeRes } from "./git-http/optimizeReq.js" +import { _checkout } from "./git/checkout.js" import type { RepoContext } from "./repoContext.js" import isoGit from "../vendored/isomorphic-git/index.js" import { checkOutPlaceholders } from "./lix/checkoutPlaceholders.js" import type { NodeishFilesystem } from "@lix-js/fs" - -const checkout = lixCheckout -// const checkout = isoGit.checkout +import { blobExistsLocaly } from "./git-http/helpers.js" export type RepoState = Awaited> @@ -31,64 +29,193 @@ export async function repoState( cache, } = ctx + const nodeishFs = withProxy({ + nodeishFs: rawFs, + verbose: debug, + description: "app", + intercept: useLazyFS ? delayedAction : undefined, + }) + + let preloads: string[] = [] + let nextBatch: (string | Promise)[] = [] + const state: { + ensureFirstBatch: (arg?: { preload?: string[] }) => Promise pending: Promise | undefined nodeishFs: NodeishFilesystem checkedOut: Set branchName: string | undefined + currentRef: string + defaultBranch: string sparseFilter: typeof args.sparseFilter } = { + ensureFirstBatch, pending: undefined, - nodeishFs: withProxy({ - nodeishFs: rawFs, - verbose: debug, - description: "app", - intercept: useLazyFS ? delayedAction : undefined, - }), + nodeishFs, checkedOut: new Set(), branchName: args.branch, + currentRef: "HEAD", + defaultBranch: "refs/remotes/origin/HEAD", sparseFilter: args.sparseFilter, } + // todo: discussion: use functions or repo state for these:?! + // state currentRef + // state baseBranch default to global base branch, other branch if on detached head on other banrch + // state defaultBranch + + // to get main base branch ref refs/remotes/origin/HEAD // Bail commit/ push on errors that are relevant or unknown - let nextBatch: string[] = [] + async function ensureFirstBatch(args?: { preload?: string[] }) { + if (!useLazyFS) { + return + } + + preloads = preloads.concat(args?.preload || []) + // ????? await checkout({ + // fs: state.nodeishFs, + // cache, + // dir, + // ref: branchName, + // filepaths: [], + // }) + + if (state.pending) { + await state.pending.catch((error) => console.error(error)) + } else { + if (preloads.length) { + nextBatch.push("") + state.pending = doCheckout().finally(() => { + state.pending = undefined + }) + } + } + } + async function doCheckout() { if (nextBatch.length < 1) { return } - const thisBatch = [...nextBatch] + const thisBatch: string[] = [] + const oidPromises: Promise[] = [] + + for (const entry of nextBatch) { + if (entry === "") { + continue + } + if (typeof entry === "string") { + if (!state.checkedOut.has(entry)) { + thisBatch.push(entry) + } + } else { + oidPromises.push(entry) + } + } + nextBatch = [] if (debug) { - console.warn("checking out ", thisBatch) + oidPromises.length && console.warn("fetching oids ", oidPromises) + } + + if (oidPromises.length > 0) { + await Promise.all(oidPromises).catch(console.error) + } + + // dedupe files which happens if git-ignores are also read or preloaded in the first batch + const allBatchFiles = [...new Set([...preloads, ...thisBatch])] + preloads = [] + + if (debug) { + console.warn("checking out ", JSON.stringify(allBatchFiles)) + } + + // FIXME: this has to be part of the checkout it self to prevent race condition!! + const oids: string[] = [] + const placeholders: string[] = allBatchFiles.filter((entry) => rawFs._isPlaceholder?.(entry)) + for (const placeholder of placeholders) { + const stats = await rawFs.stat(placeholder) + + // if (stats._rootHash) { + // FIXME: check _rootHash!!! or do this in the checkout ?!? + // for the first version this is not an issue thouhg: + // if user commits localy all files that are changed are not placeholders, all other files are the same oids as when doing original checkout + // if user fetches without checking out etc. the oids are not invalidated + // if user does a different checkout the oids are allready up to date by emptyWorkdir and fresh checkoutPlaceholders + // if user does checkout with active placehodlers wihtout removing them first, checkout will fail with dirty workdir message + // } + + oids.push(stats._oid) } - // FIXME: this has to be part of the checkout it self - to prevent race condition!! - for (const placeholder of thisBatch.filter((entry) => rawFs._isPlaceholder?.(entry))) { - await rawFs.rm(placeholder) + if (useLazyFS && oids.length > 0) { + const toFetch: string[] = ( + await Promise.all( + oids.map((oid) => + blobExistsLocaly({ + fs: rawFs, + oid, + gitdir: ".git", + }).then((exists) => (exists ? false : oid)) + ) + ) + ).filter((a) => a !== false) as string[] + + if (toFetch.length) { + // TODO: walk the oid for the paths if placeholder oid is missing or invalid + await isoGit + .fetch({ + cache, + fs: rawFs, + dir: "/", + http: makeHttpClient({ + debug, + description: "lazy fetch", + onReq: optimizeReq.bind(null, { + noBlobs: false, + addRefs: [state.branchName || "HEAD"], + overrideWants: toFetch, + }), + onRes: optimizeRes, + }), + depth: 1, + singleBranch: true, + tags: false, + }) + .catch((error: any) => { + console.error({ error, toFetch }) + }) + } } - const res = await checkout({ - fs: withProxy({ - nodeishFs: rawFs, - verbose: debug, - description: debug ? "checkout: " + JSON.stringify(thisBatch) : "checkout", - }), - dir, - cache, - ref: state.branchName, - filepaths: thisBatch, - }).catch((error: any) => { - console.error({ error, thisBatch }) - }) + let res + if (allBatchFiles.length > 0) { + await Promise.all( + placeholders.map((placeholder) => + rawFs.rm(placeholder).catch(() => { + // ignore + }) + ) + ) - for (const entry of thisBatch) { + res = await _checkout({ + fs: rawFs, + dir, + cache, + ref: state.branchName, + filepaths: allBatchFiles, + }).catch((error: any) => { + console.error({ error, allBatchFiles }) + }) + } + + for (const entry of allBatchFiles) { state.checkedOut.add(entry) } if (debug) { - console.warn("checked out ", thisBatch) + console.warn("checked out ", allBatchFiles) } if (nextBatch.length) { @@ -103,25 +230,26 @@ export async function repoState( throw new Error("fs provider does not support placeholders") } console.info("Using lix for cloning repo") - + // FIXME: symlink = false in git/config!??? await isoGit .clone({ - fs: withProxy({ nodeishFs: rawFs, verbose: debug, description: "clone" }), + fs: rawFs, http: makeHttpClient({ debug, description: "clone", - - onReq: ({ url, body }: { url: string; body: any }) => { - return optimizedRefsReq({ url, body, addRef: state.branchName }) - }, - - onRes: optimizedRefsRes, + onReq: experimentalFeatures.lazyClone + ? optimizeReq.bind(null, { + noBlobs: true, + addRefs: [state.branchName || "HEAD"], + }) + : undefined, + onRes: experimentalFeatures.lazyClone ? optimizeRes : undefined, }), dir, cache, corsProxy: gitProxyUrl, url: gitUrl, - singleBranch: true, + singleBranch: false, // if we clone with single branch true we will not get the defatult branch set in isogit noCheckout: experimentalFeatures.lazyClone, ref: state.branchName, @@ -129,15 +257,41 @@ export async function repoState( depth: 1, noTags: true, }) - .then(() => { + .then(async () => { if (!experimentalFeatures.lazyClone) { return } - return checkOutPlaceholders(ctx, { - branchName: state.branchName, - checkedOut: state.checkedOut, - sparseFilter: state.sparseFilter, - } as RepoState) + // console.log( + // "head", + // await isoGit + // .resolveRef({ fs: rawFs, dir: "/", ref: "HEAD", depth: 1 }) + // .catch(console.error) + // ) + // console.log( + // "base", + // await isoGit + // .resolveRef({ fs: rawFs, dir: "/", ref: "refs/remotes/origin/HEAD", depth: 1 }) + // .catch(console.error) + // ) + // console.log( + // "config", + // await isoGit.getConfig({ fs: rawFs, dir: "/", path: "." }).catch(console.error) + // ) + + const { gitignoreFiles } = await checkOutPlaceholders( + ctx, + { + nodeishFs: rawFs, // FIXME: state.nodeishFs, + branchName: state.branchName, + checkedOut: state.checkedOut, + sparseFilter: state.sparseFilter, + ensureFirstBatch, + } as RepoState, + { materializeGitignores: false } + ) + + // we load these on top of whatever first files are fetched in a batch, we dont need the before but need to make sure thay are available asap before workign with files + preloads = gitignoreFiles }) } else { console.info("Using existing cloned repo") @@ -166,7 +320,7 @@ export async function repoState( !state.checkedOut.has(filename) ) { if (debug) { - console.warn("delayedAction", { + console.info("delayedAction", { prop, argumentsList, rootObject, @@ -181,14 +335,89 @@ export async function repoState( // checkedOut.add(filename) // } else { - // TODO we will tackle this with the refactoring / our own implementation of checkout + // TODO: we will tackle this with the refactoring / our own implementation of checkout if (prop !== "readdir") { nextBatch.push(filename) } // } - // && nextBatch.length > 0 + if (!state.pending && nextBatch.length > 0) { + state.pending = doCheckout() + } + } else if ( + experimentalFeatures.lazyClone && + typeof rootObject !== "undefined" && + rootObject === ".git" && // TODO #1459 more solid check for git folder !filePath.startsWith(gitdir)) + pathParts[1] === "objects" && + pathParts[2] !== "pack" && + pathParts.length === 4 && + prop === "readFile" + ) { + // FIXME: is handling dir option really neceasary anywhere? + // if (dir !== undefined) { + // const dirWithLeadingSlash = dir.endsWith("/") ? dir : dir + "/" + // if (!filePath.startsWith(dirWithLeadingSlash)) { + // throw new Error( + // "Filepath " + + // filePath + + // " did not start with repo root dir " + + // dir + + // " living in git repo?" + // ) + // } + // gitFilePath = filePath.slice(dirWithLeadingSlash.length) + // } + + // we have a readFile in the .git folder - we only intercet readings on the blob files + // git checkout (called after a file was requested that doesn't exist on the client yet) + // 1. tries to read the loose object with its oid as identifier (see: https://github.com/isomorphic-git/isomorphic-git/blob/9f9ebf275520244e96c5b47df0bd5a88c514b8d2/src/storage/readObject.js#L37) + // 2. tries to find the blob in one of the downloaded pack files (see: https://github.com/isomorphic-git/isomorphic-git/blob/9f9ebf275520244e96c5b47df0bd5a88c514b8d2/src/storage/readObject.js#L37) + // if both don't exist it fill fail + // we intercept read of loose objects in 1. to check if the object exists loose or packed using blobExistsLocaly() + // if we know it doesn't exist - and also 2. would fail - we fetch the blob from remote - this will add it as a pack file and 2. will succeed + // To detect a read of a blob file we can check the path if it is an blob request and which one + // --0-- ---1--- -2 ---------------3---------------------- + // .git/objects/5d/ec81f47085ae328439d5d9e5012143aeb8fef0 + // extract the oid from the path and check if we can resolve the object loacly alread + const oid = pathParts[2] + pathParts[3] + + // FIXME: can we skip this in some situations? + nextBatch.push( + blobExistsLocaly({ + fs: rawFs, + oid, + gitdir: ".git", + }).then((existsLocaly) => { + if (!existsLocaly) { + // console.log("missing oid: ", oid) + + return isoGit.fetch({ + cache, + fs: rawFs, + dir: "/", + http: makeHttpClient({ + debug, + description: "lazy fetch", + onReq: optimizeReq.bind(null, { + noBlobs: false, + addRefs: [state.branchName || "HEAD"], + // we don't need to override the haves any more since adding the capabilities + // allow-tip-sha1-in-want allow-reachable-sha1-in-want to the request enable us to request objects explicetly + overrideWants: [oid], + }), + onRes: optimizeRes, + }), + depth: 1, + singleBranch: true, + tags: false, + }) + } + + return undefined + }) + ) + if (!state.pending) { state.pending = doCheckout() } @@ -198,7 +427,7 @@ export async function repoState( } if (state.pending) { - // TODO: move to real queue? + // move to better queue? return state.pending.then(execute).finally(() => { state.pending = undefined if (debug) { diff --git a/lix/packages/client/vendored/diff3/CHANGELOG.md b/lix/packages/client/vendored/diff3/CHANGELOG.md new file mode 100644 index 0000000000..f740360b2b --- /dev/null +++ b/lix/packages/client/vendored/diff3/CHANGELOG.md @@ -0,0 +1,4 @@ +# CHANGELOG + +## 0.0.4 +Major improvements to memory usage. diff --git a/lix/packages/client/vendored/diff3/README.md b/lix/packages/client/vendored/diff3/README.md new file mode 100644 index 0000000000..057594c946 --- /dev/null +++ b/lix/packages/client/vendored/diff3/README.md @@ -0,0 +1,28 @@ +# diff3 + +## Usage +```js +var diff3Merge = require('diff3'); +var a = ['a', 'text', 'file']; +var o = ['a', 'test', 'file']; +var b = ['a', 'toasty', 'filtered', 'file']; +var diff3 = diff3Merge(a, o, b); +``` + +## Output +```JSON +[{ + "ok": ["a"] +}, { + "conflict": { + "a": ["text"], + "aIndex": 1, + "o": ["test"], + "oIndex": 1, + "b": ["toasty", "filtered"], + "bIndex": 1 + } +}, { + "ok": ["file"] +}] +``` diff --git a/lix/packages/client/vendored/diff3/diff3.js b/lix/packages/client/vendored/diff3/diff3.js new file mode 100644 index 0000000000..6b2f7c7a46 --- /dev/null +++ b/lix/packages/client/vendored/diff3/diff3.js @@ -0,0 +1,194 @@ +// Copyright (c) 2006, 2008 Tony Garnock-Jones +// Copyright (c) 2006, 2008 LShift Ltd. +// Copyright (c) 2016, 2022 Axosoft, LLC (www.gitkraken.com) +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import onp from './onp.js'; + +function diff3MergeIndices(a, o, b) { + // Given three files, A, O, and B, where both A and B are + // independently derived from O, returns a fairly complicated + // internal representation of merge decisions it's taken. The + // interested reader may wish to consult + // + // Sanjeev Khanna, Keshav Kunal, and Benjamin C. Pierce. "A + // Formal Investigation of Diff3." In Arvind and Prasad, + // editors, Foundations of Software Technology and Theoretical + // Computer Science (FSTTCS), December 2007. + // + // (http://www.cis.upenn.edu/~bcpierce/papers/diff3-short.pdf) + var i; + + var m1 = new onp(o, a).compose(); + var m2 = new onp(o, b).compose(); + + var hunks = []; + + function addHunk(h, side) { + hunks.push([h.file1[0], side, h.file1[1], h.file2[0], h.file2[1]]); + } + for (i = 0; i < m1.length; i++) { + addHunk(m1[i], 0); + } + for (i = 0; i < m2.length; i++) { + addHunk(m2[i], 2); + } + hunks.sort(function(x, y) { + return x[0] - y[0] + }); + + var result = []; + var commonOffset = 0; + + function copyCommon(targetOffset) { + if (targetOffset > commonOffset) { + result.push([1, commonOffset, targetOffset - commonOffset]); + commonOffset = targetOffset; + } + } + + for (var hunkIndex = 0; hunkIndex < hunks.length; hunkIndex++) { + var firstHunkIndex = hunkIndex; + var hunk = hunks[hunkIndex]; + var regionLhs = hunk[0]; + var regionRhs = regionLhs + hunk[2]; + while (hunkIndex < hunks.length - 1) { + var maybeOverlapping = hunks[hunkIndex + 1]; + var maybeLhs = maybeOverlapping[0]; + if (maybeLhs > regionRhs) break; + regionRhs = Math.max(regionRhs, maybeLhs + maybeOverlapping[2]); + hunkIndex++; + } + + copyCommon(regionLhs); + if (firstHunkIndex == hunkIndex) { + // The "overlap" was only one hunk long, meaning that + // there's no conflict here. Either a and o were the + // same, or b and o were the same. + if (hunk[4] > 0) { + result.push([hunk[1], hunk[3], hunk[4]]); + } + } else { + // A proper conflict. Determine the extents of the + // regions involved from a, o and b. Effectively merge + // all the hunks on the left into one giant hunk, and + // do the same for the right; then, correct for skew + // in the regions of o that each side changed, and + // report appropriate spans for the three sides. + var regions = { + 0: [a.length, -1, o.length, -1], + 2: [b.length, -1, o.length, -1] + }; + for (i = firstHunkIndex; i <= hunkIndex; i++) { + hunk = hunks[i]; + var side = hunk[1]; + var r = regions[side]; + var oLhs = hunk[0]; + var oRhs = oLhs + hunk[2]; + var abLhs = hunk[3]; + var abRhs = abLhs + hunk[4]; + r[0] = Math.min(abLhs, r[0]); + r[1] = Math.max(abRhs, r[1]); + r[2] = Math.min(oLhs, r[2]); + r[3] = Math.max(oRhs, r[3]); + } + var aLhs = regions[0][0] + (regionLhs - regions[0][2]); + var aRhs = regions[0][1] + (regionRhs - regions[0][3]); + var bLhs = regions[2][0] + (regionLhs - regions[2][2]); + var bRhs = regions[2][1] + (regionRhs - regions[2][3]); + result.push([-1, + aLhs, aRhs - aLhs, + regionLhs, regionRhs - regionLhs, + bLhs, bRhs - bLhs + ]); + } + commonOffset = regionRhs; + } + + copyCommon(o.length); + return result; +} + +function diff3Merge(a, o, b) { + // Applies the output of Diff.diff3_merge_indices to actually + // construct the merged file; the returned result alternates + // between "ok" and "conflict" blocks. + + var result = []; + var files = [a, o, b]; + var indices = diff3MergeIndices(a, o, b); + + var okLines = []; + + function flushOk() { + if (okLines.length) { + result.push({ + ok: okLines + }); + } + okLines = []; + } + + function pushOk(xs) { + for (const x_ of xs) { + okLines.push(x_); + } + } + + function isTrueConflict(rec) { + if (rec[2] != rec[6]) return true; + var aoff = rec[1]; + var boff = rec[5]; + for (var j = 0; j < rec[2]; j++) { + if (a[j + aoff] != b[j + boff]) return true; + } + return false; + } + + for (var x of indices) { + var side = x[0]; + if (side == -1) { + if (!isTrueConflict(x)) { + pushOk(files[0].slice(x[1], x[1] + x[2])); + } else { + flushOk(); + result.push({ + conflict: { + a: a.slice(x[1], x[1] + x[2]), + aIndex: x[1], + o: o.slice(x[3], x[3] + x[4]), + oIndex: x[3], + b: b.slice(x[5], x[5] + x[6]), + bIndex: x[5] + } + }); + } + } else { + pushOk(files[side].slice(x[1], x[1] + x[2])); + } + } + + flushOk(); + return result; +} + +export default diff3Merge; diff --git a/lix/packages/client/vendored/diff3/onp.js b/lix/packages/client/vendored/diff3/onp.js new file mode 100644 index 0000000000..6d4e1de431 --- /dev/null +++ b/lix/packages/client/vendored/diff3/onp.js @@ -0,0 +1,152 @@ +/* + * URL: https://github.com/cubicdaiya/onp + * + * Copyright (c) 2013 Tatsuhiko Kubo + * Copyright (c) 2016, 2022 Axosoft, LLC (www.gitkraken.com) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * The algorithm implemented here is based on "An O(NP) Sequence Comparison Algorithm" + * by described by Sun Wu, Udi Manber and Gene Myers +*/ +export default function (a_, b_) { + var a = a_, + b = b_, + m = a.length, + n = b.length, + reverse = false, + offset = m + 1, + path = [], + pathposi = []; + + var tmp1, + tmp2; + + var init = function () { + if (m >= n) { + tmp1 = a; + tmp2 = m; + a = b; + b = tmp1; + m = n; + n = tmp2; + reverse = true; + offset = m + 1; + } + }; + + var P = function (startX, startY, endX, endY, r) { + return { + startX, + startY, + endX, + endY, + r + }; + }; + + var snake = function (k, p, pp) { + var r, x, y, startX, startY; + if (p > pp) { + r = path[k-1+offset]; + } else { + r = path[k+1+offset]; + } + + startY = y = Math.max(p, pp); + startX = x = y - k; + while (x < m && y < n && a[x] === b[y]) { + ++x; + ++y; + } + + if (startX == x && startY == y) { + path[k+offset] = r; + } else { + path[k+offset] = pathposi.length; + pathposi[pathposi.length] = new P(startX, startY, x, y, r); + } + return y; + }; + + init(); + + return { + compose : function () { + var delta, size, fp, p, r, i, k, lastStartX, lastStartY, result; + delta = n - m; + size = m + n + 3; + fp = {}; + for (i=0;i=delta+1;--k) { + fp[k+offset] = snake(k, fp[k-1+offset]+1, fp[k+1+offset]); + } + fp[delta+offset] = snake(delta, fp[delta-1+offset]+1, fp[delta+1+offset]); + } while (fp[delta+offset] !== n); + + // THIS IS PATCHED BY LIX: const ed = delta + 2 * p; + + r = path[delta+offset]; + lastStartX = m; + lastStartY = n; + result = []; + while (r !== -1) { + let elem = pathposi[r]; + if (m != elem.endX || n != elem.endY) { + result.push({ + file1: [ + reverse ? elem.endY : elem.endX, + reverse ? lastStartY - elem.endY : lastStartX - elem.endX + ], + file2: [ + reverse ? elem.endX : elem.endY, + reverse ? lastStartX - elem.endX : lastStartY - elem.endY + ] + }); + } + + lastStartX = elem.startX; + lastStartY = elem.startY; + + r = pathposi[r].r; + } + + if (lastStartX != 0 || lastStartY != 0) { + result.push({ + file1: [0, reverse ? lastStartY : lastStartX], + file2: [0, reverse ? lastStartX : lastStartY] + }) + } + + result.reverse(); + return result; + } + }; +}; diff --git a/lix/packages/client/vendored/diff3/package.json b/lix/packages/client/vendored/diff3/package.json new file mode 100644 index 0000000000..401be4145d --- /dev/null +++ b/lix/packages/client/vendored/diff3/package.json @@ -0,0 +1,32 @@ +{ + "name": "diff3", + "version": "0.0.4", + "description": "A diff3 engine for nodejs, converted to mdoule by lix and fixed 1 line in onp.js", + "main": "diff3.js", + "type": "module", + "exports": { + ".": "./diff3.js", + "./onp.js": "./onp.js" + }, + "files": [ + "diff3.js", + "onp.js" + ], + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/axosoft/diff3.git" + }, + "keywords": [ + "diff3", + "diff" + ], + "author": "Tyler Wanek & Jacob Watson", + "license": "MIT", + "bugs": { + "url": "https://github.com/axosoft/diff3/issues" + }, + "homepage": "https://github.com/axosoft/diff3#readme" +} \ No newline at end of file diff --git a/lix/packages/client/vendored/isomorphic-git/index.js b/lix/packages/client/vendored/isomorphic-git/index.js index 7150465f39..0968cb9eb8 100644 --- a/lix/packages/client/vendored/isomorphic-git/index.js +++ b/lix/packages/client/vendored/isomorphic-git/index.js @@ -5133,7 +5133,7 @@ async function addToIndex({ * * @returns {Promise} Resolves successfully with the SHA-1 object id of the newly created commit. */ -async function _commit({ +async function _commit ({ fs, cache, onSign, diff --git a/lix/packages/dev.sh b/lix/packages/dev.sh new file mode 100755 index 0000000000..11885bad42 --- /dev/null +++ b/lix/packages/dev.sh @@ -0,0 +1 @@ +pnpm --filter=@lix-js/client --filter=@lix-js/fs --filter=exp install && pnpm --parallel --filter=@lix-js/client --filter=@lix-js/fs --filter=exp dev \ No newline at end of file diff --git a/lix/packages/docker-compose.yaml b/lix/packages/docker-compose.yaml index e89090a605..4aa7033b26 100644 --- a/lix/packages/docker-compose.yaml +++ b/lix/packages/docker-compose.yaml @@ -1,12 +1,13 @@ -version: '3.7' +# develop with docker-compose watch in this folder + name: lix services: git: restart: always container_name: git - image: git:10 + image: git:12 hostname: git - ports: [ "8000" ] + # ports: [ "8000" ] labels: [ dev.orbstack.domains=git.local ] entrypoint: /start.sh # stdin_open: true @@ -17,19 +18,49 @@ services: dockerfile: Dockerfile networks: - lix - # depends_on: - # - exp - # image downgrades user but that seems not to work, see: https://caddy.community/t/basic-docker-compose-setup-failing/6892/7?u=alexander_gabriel user: root # env_file: # - ./.env + gitea: + labels: [ dev.orbstack.domains=gitea.local ] + image: gitea/gitea:1.21.11 + container_name: gitea + hostname: gitea + environment: + - USER_UID=1000 + - USER_GID=1000 + restart: always + networks: + - lix + volumes: + - ./gitea:/data + # - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro + # ports: + # - "3000:3000" + + runner: + restart: always + container_name: gitea-runner + hostname: runner + image: gitea/act_runner:nightly + environment: + CONFIG_FILE: /config.yaml + GITEA_INSTANCE_URL: "http://gitea.local:3000" + GITEA_RUNNER_REGISTRATION_TOKEN: "pb3VkhEXa03IaQPUFSGPP2nImshQYvr72d1ZYH3Z" + GITEA_RUNNER_NAME: "local" + GITEA_RUNNER_LABELS: "" + volumes: + - ./gitea/runner/config.yaml:/config.yaml + - ./gitea/runner:/data + - /var/run/docker.sock:/var/run/docker.sock + exp: restart: always container_name: exp hostname: exp labels: [ dev.orbstack.domains=exp.local ] - image: exp:4 + image: exp:6 build: dockerfile_inline: | FROM node:20-slim @@ -37,15 +68,12 @@ services: RUN apt-get install openssl curl libssl-dev -y RUN npm -g install pnpm COPY --chown=app:app . /workspace + RUN chmod +x /workspace/dev.sh WORKDIR /workspace - RUN pnpm --filter=@lix-js/client --filter=@lix-js/fs --filter=lix-app install - RUN pnpm --filter=@lix-js/client --filter=@lix-js/fs --filter=lix-app build - # todo: shell aliase - ports: [ "3334" ] - working_dir: /workspace/exp - entrypoint: pnpm dev - # working_dir: /workspace/exp + # ports: [ "3334" ] + working_dir: /workspace + entrypoint: /bin/bash ./dev.sh # entrypoint: /bin/bash # stdin_open: true # tty: true @@ -87,5 +115,4 @@ networks: driver: bridge volumes: - josh-vol: {} pnpm-store: {} diff --git a/lix/packages/exp/index.html b/lix/packages/exp/index.html index b6c5f0afaf..71faa6cbc6 100644 --- a/lix/packages/exp/index.html +++ b/lix/packages/exp/index.html @@ -8,6 +8,10 @@
+ +
+ + diff --git a/lix/packages/exp/package.json b/lix/packages/exp/package.json index 5e1d8ed9db..a43a8976e4 100644 --- a/lix/packages/exp/package.json +++ b/lix/packages/exp/package.json @@ -1,5 +1,5 @@ { - "name": "exp-app", + "name": "exp", "version": "0.0.1", "type": "module", "scripts": { @@ -9,16 +9,17 @@ "_check": "svelte-check --tsconfig ./tsconfig.json" }, "devDependencies": { - "@sveltejs/vite-plugin-svelte": "3.0.2", - "@tsconfig/svelte": "5.0.2", + "@sveltejs/vite-plugin-svelte": "3.1.0", + "@tsconfig/svelte": "5.0.4", "svelte": "5.0.0-next.80", "svelte-check": "3.6.0", "tslib": "2.6.2", "typescript": "5.2.2", - "vite": "5.1.1" + "vite": "5.2.11" }, "dependencies": { "@aws-sdk/client-s3": "^3.533.0", + "@gitgraph/js": "^1.4.0", "@helia/unixfs": "3.0.2", "@lix-js/client": "../client", "@lix-js/fs": "../fs", diff --git a/lix/packages/exp/src/App.svelte b/lix/packages/exp/src/App.svelte index cf00849a55..21181e769c 100644 --- a/lix/packages/exp/src/App.svelte +++ b/lix/packages/exp/src/App.svelte @@ -1,20 +1,22 @@
@@ -77,6 +158,7 @@