forked from shioju/crawlee-core
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cookie_utils.js
119 lines (119 loc) · 4.6 KB
/
cookie_utils.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getCookiesFromResponse = getCookiesFromResponse;
exports.getDefaultCookieExpirationDate = getDefaultCookieExpirationDate;
exports.toughCookieToBrowserPoolCookie = toughCookieToBrowserPoolCookie;
exports.browserPoolCookieToToughCookie = browserPoolCookieToToughCookie;
exports.cookieStringToToughCookie = cookieStringToToughCookie;
exports.mergeCookies = mergeCookies;
const tough_cookie_1 = require("tough-cookie");
const log_1 = require("./log");
const errors_1 = require("./session_pool/errors");
/**
* @internal
*/
function getCookiesFromResponse(response) {
const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
const cookieHeader = headers['set-cookie'] || '';
try {
return Array.isArray(cookieHeader)
? cookieHeader.map((cookie) => tough_cookie_1.Cookie.parse(cookie))
: [tough_cookie_1.Cookie.parse(cookieHeader)];
}
catch (e) {
throw new errors_1.CookieParseError(cookieHeader);
}
}
/**
* Calculate cookie expiration date
* @param maxAgeSecs
* @returns Calculated date by session max age seconds.
* @internal
*/
function getDefaultCookieExpirationDate(maxAgeSecs) {
return new Date(Date.now() + maxAgeSecs * 1000);
}
/**
* Transforms tough-cookie to puppeteer cookie.
* @param toughCookie Cookie from CookieJar
* @return Cookie compatible with browser pool
* @internal
*/
function toughCookieToBrowserPoolCookie(toughCookie) {
return {
name: toughCookie.key,
value: toughCookie.value,
// Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
// If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
expires: toughCookie.expires === 'Infinity' ? undefined : new Date(toughCookie.expires).getTime() / 1000,
domain: toughCookie.domain ? `${toughCookie.hostOnly ? '' : '.'}${toughCookie.domain}` : undefined,
path: toughCookie.path ?? undefined,
secure: toughCookie.secure,
httpOnly: toughCookie.httpOnly,
};
}
/**
* Transforms browser-pool cookie to tough-cookie.
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
* @internal
*/
function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
const expires = isExpiresValid
? new Date(cookieObject.expires * 1000)
: getDefaultCookieExpirationDate(maxAgeSecs);
const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
return new tough_cookie_1.Cookie({
key: cookieObject.name,
value: cookieObject.value,
expires,
domain,
path: cookieObject.path,
secure: cookieObject.secure,
httpOnly: cookieObject.httpOnly,
hostOnly: !domainHasLeadingDot,
});
}
/**
* @internal
* @param cookieString The cookie string to attempt parsing
* @returns Browser pool compatible cookie, or null if cookie cannot be parsed
*/
function cookieStringToToughCookie(cookieString) {
const parsed = tough_cookie_1.Cookie.parse(cookieString);
if (parsed) {
return toughCookieToBrowserPoolCookie(parsed);
}
return null;
}
/**
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
* if we see two cookies with same keys but different casing.
* @internal
*/
function mergeCookies(url, sourceCookies) {
const jar = new tough_cookie_1.CookieJar();
// ignore empty cookies
for (const sourceCookieString of sourceCookies) {
// ignore empty cookies
if (!sourceCookieString)
continue;
const cookies = sourceCookieString.split(/ *; */);
for (const cookieString of cookies) {
// ignore extra spaces
if (!cookieString)
continue;
const cookie = tough_cookie_1.Cookie.parse(cookieString);
const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
});
if (similarKeyCookie) {
log_1.log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
}
jar.setCookieSync(cookie, url);
}
}
return jar.getCookieStringSync(url);
}
//# sourceMappingURL=cookie_utils.js.map