Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create raw data for cache invalidation report #9422

Open
wants to merge 14 commits into
base: v2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/core/core/src/Parcel.js
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,9 @@ export default class Parcel {
createValidationRequest({optionsRef: this.#optionsRef, assetRequests}),
{force: assetRequests.length > 0},
);
if (process.env.PARCEL_GENERATE_INVALIDATION_REPORT != null) {
this.#requestTracker.generateInvalidationReport();
}
return event;
} catch (e) {
if (e instanceof BuildAbortError) {
Expand Down
95 changes: 86 additions & 9 deletions packages/core/core/src/RequestTracker.js
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ type Request<TInput, TResult> = {|
|};

type InvalidateReason = number;
type InvalidationEntry = {|
invalidated: NodeId,
cause?: NodeId,
|};

type RequestNode = {|
id: ContentKey,
+type: typeof REQUEST,
Expand Down Expand Up @@ -233,6 +238,7 @@ export class RequestGraph extends ContentGraph<
RequestGraphEdgeType,
> {
invalidNodeIds: Set<NodeId> = new Set();
invalidationReport: InvalidationEntry[] = [];
incompleteNodeIds: Set<NodeId> = new Set();
incompleteNodePromises: Map<NodeId, Promise<boolean>> = new Map();
globNodeIds: Set<NodeId> = new Set();
Expand Down Expand Up @@ -332,18 +338,21 @@ export class RequestGraph extends ContentGraph<
);
}

invalidateNode(nodeId: NodeId, reason: InvalidateReason) {
invalidateNode(nodeId: NodeId, reason: InvalidateReason, cause?: NodeId) {
let node = nullthrows(this.getNode(nodeId));
invariant(node.type === REQUEST);
node.invalidateReason |= reason;
this.invalidNodeIds.add(nodeId);
if (process.env.PARCEL_GENERATE_INVALIDATION_REPORT != null) {
this.invalidationReport.push({invalidated: nodeId, cause: cause});
}

let parentNodes = this.getNodeIdsConnectedTo(
nodeId,
requestGraphEdgeTypes.subrequest,
);
for (let parentNode of parentNodes) {
this.invalidateNode(parentNode, reason);
this.invalidateNode(parentNode, reason, nodeId);
}
}

Expand Down Expand Up @@ -373,7 +382,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_update,
);
for (let parentNode of parentNodes) {
this.invalidateNode(parentNode, ENV_CHANGE);
this.invalidateNode(parentNode, ENV_CHANGE, nodeId);
}
}
}
Expand All @@ -391,7 +400,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_update,
);
for (let parentNode of parentNodes) {
this.invalidateNode(parentNode, OPTION_CHANGE);
this.invalidateNode(parentNode, OPTION_CHANGE, nodeId);
}
}
}
Expand Down Expand Up @@ -710,7 +719,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_create,
);
for (let connectedNode of connectedNodes) {
this.invalidateNode(connectedNode, FILE_CREATE);
this.invalidateNode(connectedNode, FILE_CREATE, matchNodeId);
}
}
}
Expand Down Expand Up @@ -770,7 +779,7 @@ export class RequestGraph extends ContentGraph<

for (let connectedNode of nodes) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_UPDATE);
this.invalidateNode(connectedNode, FILE_UPDATE, nodeId);
}

if (type === 'create') {
Expand All @@ -780,7 +789,7 @@ export class RequestGraph extends ContentGraph<
);
for (let connectedNode of nodes) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_CREATE);
this.invalidateNode(connectedNode, FILE_CREATE, nodeId);
}
}
} else if (type === 'create') {
Expand Down Expand Up @@ -821,7 +830,7 @@ export class RequestGraph extends ContentGraph<
);
for (let connectedNode of connectedNodes) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_CREATE);
this.invalidateNode(connectedNode, FILE_CREATE, globeNodeId);
}
}
}
Expand All @@ -832,7 +841,7 @@ export class RequestGraph extends ContentGraph<
requestGraphEdgeTypes.invalidated_by_delete,
)) {
didInvalidate = true;
this.invalidateNode(connectedNode, FILE_DELETE);
this.invalidateNode(connectedNode, FILE_DELETE, nodeId);
}

// Delete the file node since it doesn't exist anymore.
Expand Down Expand Up @@ -1108,6 +1117,74 @@ export default class RequestTracker {
return {api, subRequestContentKeys};
}

generateInvalidationReport(): void {
let nodes = {};
let invalidationRelations = {};
for (let {invalidated, cause} of this.graph.invalidationReport) {
let invalidatedNode = this.graph.getNode(invalidated);
let causeNode =
cause !== undefined ? this.graph.getNode(cause) : undefined;
if (
invalidatedNode === null ||
causeNode === null ||
invalidatedNode === undefined ||
causeNode === undefined
) {
continue;
}

if (invalidatedNode.id !== 'Main') {
if (!invalidationRelations[invalidated]) {
invalidationRelations[invalidated] = [];
}

if (!invalidationRelations[invalidated].includes(cause)) {
invariant(cause !== undefined);
invalidationRelations[invalidated].push(cause);
}
}

invariant(invalidatedNode.type === 1);
nodes[invalidated] = {
id: invalidatedNode.id,
type: invalidatedNode.type,
requestType: invalidatedNode.requestType,
invalidateReason: invalidatedNode.invalidateReason,
};

invariant(cause !== undefined);
if (
causeNode.requestType !== undefined &&
causeNode.invalidateReason !== undefined
) {
nodes[cause] = {
id: causeNode.id,
type: causeNode.type,
requestType: causeNode.requestType,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The types have been all migrated into numbers now right? Maybe we should map them back to strings for the report?

invalidateReason: causeNode.invalidateReason,
};
} else {
nodes[cause] = {
id: causeNode.id,
type: causeNode.type,
};
}
}

const fs = require('fs');
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Parcel has the ability to override the output filesystem which this ignores. Would it be hard to use outputFS instead of requiring the node FS directly?

fs.writeFile(
'cache-invalidation-report.json',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe should prefix this with parcel? Also we should probably ensure the output is relative to the project root.

JSON.stringify(
{invalidationRelations: invalidationRelations, nodes: nodes},
undefined,
4,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe 2? #nitpick 😅

),
function (err) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is running async via a callback but we're not waiting for completion anywhere. We should be awaiting a promise here instead. If you migrate to outputFS then this should be easy.

if (err) throw err;
},
);
}

async writeToCache() {
let cacheKey = getCacheKey(this.options);
let requestGraphKey =
Expand Down
Loading