Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Create raw data for cache invalidation report #9422

Open
wants to merge 14 commits into
base: v2
Choose a base branch
from
Open

Create raw data for cache invalidation report #9422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
72bbd1e
Generate initial cache report
irismoini Nov 30, 2023
1365931
Generate cache invalidation report post build
irismoini Dec 4, 2023
95effa3
Update generateInvalidationReport for node type check
irismoini Dec 6, 2023
88c02d7
Make flow happy
irismoini Dec 6, 2023
dec4823
Fix white space
irismoini Dec 7, 2023
cdad594
Merge branch 'v2' into imoini/P2X-1144-add-logging-for-cache-invalida…
irismoini Dec 7, 2023
a919a60
Merge branch 'v2' into imoini/P2X-1144-add-logging-for-cache-invalida…
irismoini Dec 7, 2023
0e993c5
Put invalidation report behind flag
irismoini Dec 7, 2023
36c24f1
Merge branch 'v2' into imoini/P2X-1144-add-logging-for-cache-invalida…
irismoini Dec 12, 2023
7194dbc
Put invalidation report behind flag
irismoini Dec 13, 2023
0231d44
Merge branch 'v2' into imoini/P2X-1144-add-logging-for-cache-invalida…
irismoini Dec 13, 2023
b133f00
Merge branch 'v2' into imoini/P2X-1144-add-logging-for-cache-invalida…
irismoini Dec 14, 2023
578ea28
Merge branch 'v2' into imoini/P2X-1144-add-logging-for-cache-invalida…
irismoini Dec 14, 2023
c7d2677
Merge branch 'v2' into imoini/P2X-1144-add-logging-for-cache-invalida…
irismoini Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/core/core/src/Parcel.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,9 @@ export default class Parcel {
createValidationRequest({optionsRef: this.#optionsRef, assetRequests}),
{force: assetRequests.length > 0},
);
if (process.env.PARCEL_GENERATE_INVALIDATION_REPORT != null) {
this.#requestTracker.generateInvalidationReport();
}
return event;
} catch (e) {
if (e instanceof BuildAbortError) {
Expand Down
95 changes: 86 additions & 9 deletions packages/core/core/src/RequestTracker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ type Request<TInput, TResult> = {|
|};

type InvalidateReason = number;
type InvalidationEntry = {|
invalidated: NodeId,
cause?: NodeId,
|};

type RequestNode = {|
id: ContentKey,
+type: typeof REQUEST,
Expand Down Expand Up @@ -233,6 +238,7 @@ export class RequestGraph extends ContentGraph<
RequestGraphEdgeType,
> {
invalidNodeIds: Set<NodeId> = new Set();
invalidationReport: InvalidationEntry[] = [];
incompleteNodeIds: Set<NodeId> = new Set();
incompleteNodePromises: Map<NodeId, Promise<boolean>> = new Map();
globNodeIds: Set<NodeId> = new Set();
Expand Down Expand Up @@ -332,18 +338,21 @@ export class RequestGraph extends ContentGraph<
);
}

invalidateNode(nodeId: NodeId, reason: InvalidateReason) {
invalidateNode(nodeId: NodeId, reason: InvalidateReason, cause?: NodeId) {
let node = nullthrows(this.getNode(nodeId));
invariant(node.type === REQUEST);
node.invalidateReason |= reason;
this.invalidNodeIds.add(nodeId);
if (process.env.PARCEL_GENERATE_INVALIDATION_REPORT != null) {
this.invalidationReport.push({invalidated: nodeId, cause: cause});
}

let parentNodes = this.getNodeIdsConnectedTo(
nodeId,
requestGraphEdgeTypes.subrequest,
);
for (let parentNode of parentNodes) {
this.invalidateNode(parentNode, reason);
this.invalidateNode(parentNode, reason, nodeId);
}
}

Expand Down Expand Up @@ -373,7 +382,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_update,
);
for (let parentNode of parentNodes) {
this.invalidateNode(parentNode, ENV_CHANGE);
this.invalidateNode(parentNode, ENV_CHANGE, nodeId);
}
}
}
Expand All @@ -391,7 +400,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_update,
);
for (let parentNode of parentNodes) {
this.invalidateNode(parentNode, OPTION_CHANGE);
this.invalidateNode(parentNode, OPTION_CHANGE, nodeId);
}
}
}
Expand Down Expand Up @@ -710,7 +719,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_create,
);
for (let connectedNode of connectedNodes) {
this.invalidateNode(connectedNode, FILE_CREATE);
this.invalidateNode(connectedNode, FILE_CREATE, matchNodeId);
}
}
}
Expand Down Expand Up @@ -770,7 +779,7 @@ export class RequestGraph extends ContentGraph<

for (let connectedNode of nodes) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_UPDATE);
this.invalidateNode(connectedNode, FILE_UPDATE, nodeId);
}

if (type === 'create') {
Expand All @@ -780,7 +789,7 @@ export class RequestGraph extends ContentGraph<
);
for (let connectedNode of nodes) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_CREATE);
this.invalidateNode(connectedNode, FILE_CREATE, nodeId);
}
}
} else if (type === 'create') {
Expand Down Expand Up @@ -821,7 +830,7 @@ export class RequestGraph extends ContentGraph<
);
for (let connectedNode of connectedNodes) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_CREATE);
this.invalidateNode(connectedNode, FILE_CREATE, globeNodeId);
}
}
}
Expand All @@ -832,7 +841,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_delete,
)) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_DELETE);
this.invalidateNode(connectedNode, FILE_DELETE, nodeId);
}

// Delete the file node since it doesn't exist anymore.
Expand Down Expand Up @@ -1108,6 +1117,74 @@ export default class RequestTracker {
return {api, subRequestContentKeys};
}

generateInvalidationReport(): void {
let nodes = {};
let invalidationRelations = {};
for (let {invalidated, cause} of this.graph.invalidationReport) {
let invalidatedNode = this.graph.getNode(invalidated);
let causeNode =
cause !== undefined ? this.graph.getNode(cause) : undefined;
if (
invalidatedNode === null ||
causeNode === null ||
invalidatedNode === undefined ||
causeNode === undefined
) {
continue;
}

if (invalidatedNode.id !== 'Main') {
if (!invalidationRelations[invalidated]) {
invalidationRelations[invalidated] = [];
}

if (!invalidationRelations[invalidated].includes(cause)) {
invariant(cause !== undefined);
invalidationRelations[invalidated].push(cause);
}
}

invariant(invalidatedNode.type === 1);
nodes[invalidated] = {
id: invalidatedNode.id,
type: invalidatedNode.type,
requestType: invalidatedNode.requestType,
invalidateReason: invalidatedNode.invalidateReason,
};

invariant(cause !== undefined);
if (
causeNode.requestType !== undefined &&
causeNode.invalidateReason !== undefined
) {
nodes[cause] = {
id: causeNode.id,
type: causeNode.type,
requestType: causeNode.requestType,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The types have been all migrated into numbers now right? Maybe we should map them back to strings for the report?

invalidateReason: causeNode.invalidateReason,
};
} else {
nodes[cause] = {
id: causeNode.id,
type: causeNode.type,
};
}
}

const fs = require('fs');
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Parcel has the ability to override the output filesystem which this ignores. Would it be hard to use outputFS instead of requiring the node FS directly?

fs.writeFile(
'cache-invalidation-report.json',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe should prefix this with parcel? Also we should probably ensure the output is relative to the project root.

JSON.stringify(
{invalidationRelations: invalidationRelations, nodes: nodes},
undefined,
4,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe 2? #nitpick 😅

),
function (err) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is running async via a callback but we're not waiting for completion anywhere. We should be awaiting a promise here instead. If you migrate to outputFS then this should be easy.

if (err) throw err;
},
);
}

async writeToCache() {
let cacheKey = getCacheKey(this.options);
let requestGraphKey =
Expand Down