Repository navigation
Expand file tree
/
Copy pathfunction.js
More file actions
171 lines (138 loc) · 7.85 KB
/
Copy pathfunction.js
File metadata and controls
171 lines (138 loc) · 7.85 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
function handler(event) {
const request = event.request;
// rationale: README.md#allowlisted-uris
if (allowlistedUriRegex.test(request.uri || '')) {
return request;
}
// Block requests with no user agent (cheap check, done before any URI decoding)
const userAgentHeader = request.headers['user-agent'];
if (!userAgentHeader || !userAgentHeader.value || !userAgentHeader.value.trim()) {
return createBlockedResponse(request.uri || '');
}
const uri = decodeUri(request.uri || '');
if (uri === null) {
return createNotFoundResponse();
}
// Lowercased copy for case-insensitive pattern matching (UA, file extensions, etc.)
const uriLower = uri.trim().toLowerCase();
const ua = userAgentHeader.value.toLowerCase();
// rationale: README.md#bad-actor-response-mapping
if (isBadActor(uriLower, ua) ||
isBlockedAsn(request.headers['cloudfront-viewer-asn'], ua) || isBlockedBot(ua, uriLower)) {
return createBlockedResponse(uriLower);
}
if (isDisallowedAsset(uriLower) || isCoverDenied(uriLower, request.headers.referer)) {
return createNotFoundResponse();
}
return request;
}
// rationale: README.md#uri-decoding
function decodeUri(uri) {
try {
for (let i = 0; i < 3 && uri.indexOf('%') !== -1; i++) {
const decoded = decodeURIComponent(uri);
if (decoded === uri) break;
uri = decoded;
}
} catch (_e) {
return null;
}
return uri;
}
// rationale: README.md#json-allowlist
// rationale: README.md#js-allowlist
function isDisallowedAsset(uri) {
return (uri.slice(-5) === '.json' && !allowedJsonRegex.test(uri)) ||
(uri.slice(-3) === '.js' && !allowedJsRegex.test(uri));
}
// rationale: README.md#cover-referer
function isCoverDenied(uri, referer) {
return coverUriRegex.test(uri) &&
(!referer || !referer.value || referer.value.toLowerCase().indexOf(COVER_REFERER_PREFIX) !== 0);
}
// rationale: README.md#cover-referer
const coverUriRegex = /^\/assets\/posts(_640)?\/.+\.webp$/;
const COVER_REFERER_PREFIX = 'https://post-tenebras-lire.net/';
// rationale: README.md#json-allowlist
const allowedJsonRegex = /^\/(about\/data\/(blogs|pages|visitors)|human|pagefind\/pagefind-entry)\.json$/;
// rationale: README.md#allowlisted-uris
const allowlistedUriRegex = /^\/(backup\.zip)$/i;
// rationale: README.md#js-allowlist
const allowedJsRegex = /^\/(javascript\/(recommended-blogs|chart\.umd\.min|bootstrap\.bundle\.min)|pagefind\/pagefind(-worker|-ui)?)\.js$/;
// rationale: README.md#bad-actor-check-order
function isBadActor(uri, ua) {
return isPathTraversal(uri) ||
isDotfilePath(uri) ||
isSecurityScanUri(uri) ||
isTruncatedChromeUA(ua) ||
isMalformedChromeClaim(ua);
}
// rationale: README.md#dotfile-path
const dotfilePathRegex = /\/\./;
function isDotfilePath(uri) {
return dotfilePathRegex.test(uri);
}
// rationale: README.md#security-scan-regex
const securityScanRegex = /\.(php\d*|sql|bak|swp|phtml|config|ya?ml|toml|conf|key|pem|axd|boto|s3cfg|htpasswd|tfstate|old|env|map|webmanifest)$|~$|^\/(images?|img|wp-includes|wp-content|wp-json|static|wp|wordpress|old|new|blog|backup|cgi-bin|admin|administrator|wp-admin|phpmyadmin|vendor|uploads|plugins|login|webmail|roundcube|mail|rc|actuator|api|@fs|@vite|userfiles|telescope|horizon|storage|debug|console|server-status|server-info|manage|graphql|v1|health|proc|var|dockerfile)(\/|$)|^\/(secrets?|config|credentials?|service[-_]account|firebase-(?:adminsdk|service-account|config)|serviceaccountkey|settings|env|auth|app-config|appsettings|openapi|swagger|amplifyconfiguration)\.json$|^\/(_|id_)/;
function isSecurityScanUri(uri) {
return uri === '/ip' || securityScanRegex.test(uri);
}
// rationale: README.md#path-traversal
function isPathTraversal(uri) {
return uri.indexOf('..') !== -1;
}
// rationale: README.md#truncated-chrome-ua
function isTruncatedChromeUA(ua) {
return ua === 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36';
}
// rationale: README.md#malformed-chrome-claim
function isMalformedChromeClaim(ua) {
return ua.indexOf('chrome/') !== -1 && ua.indexOf('applewebkit') === -1;
}
// rationale: README.md#blocked-bot-regex
const blockedBotRegex = /linkupbot\/|sleepbot|got \(https:\/\/github\.com\/sindresorhus\/got|palo alto networks|trident|amzn-searchbot\/|ccbot\/|aiohttp\/|emacs\/|meta-webindexer\/|twitterbot\/1\.0|presto|analyseseonet\/|scrapy|headlesschrome|aranea web-crawled corpora project|pimeyes-downloader-api|bytespider|python-httpx\/|mach-o|intelx\.io_bot|welley\/|searchenginebot|python-requests\/|databankmetasearch|shapbot|cms-detector\/|navcrawl\/|wellknownbot|siteauditbot\/|ev-crawler|builtwith|timpibot|xai-searchbot\/|semrushbot|livelapbot\/|engagemiibot\/|sitescan\/|stackyenrich\/|testsearchspider|atlas-enrich\/|fyndbot|wpbot\/|googlebot|dataforseobot|imagebot\/|loadedbot\/|google-cloudvertexbot|googleother|koofie\.net\/|domain-intel\/|screaming frog seo spider|openclaw|discordbot\/|sharkey \(like|reflectionbot\/|lightpanda\/|forestengine\/|seojuice-searchbot\/|coccocbot|hubspot crawler|domain-harvester\/|mapthenetbot\/|expansel-monitor\/|fogbot\/|newsletterformresearchbot\/|srchs-research-bot\/|aionbot\/|tiktokspider|opentheboxbot\/|veryhip\/|cms-security-auditor\/|censysinspect\/|publicwwwbot\/|wp2shell|webatlabot|ssi-nutch\/|variableratio-publicassetresearch\/|halobot\/|flowb0t-contentengine\/|claritybot\/|undici|serpex-index\/|compatible; crawler\)|webapp-mapper\/|ironfountain-leads\/|what10bot\/|konqueror\/|brightbot|rust-scrape\/|crawl-engine\/|keenablebot\/|wordpress\/|curl\/|tphotobot\/|websiteresearch\/|surdotlybot\/|aonsightmetabot\/|baiduspider|sqlspider\/|goscan-mini\/|storebot-google\/|googleimageproxy|app::jupiter\/|httpx - open-source project/;
function isBlockedBot(normalizedUserAgent, uri) {
// rationale: README.md#feedfetcher-google
if (normalizedUserAgent.indexOf('feedfetcher-google') !== -1) {
return uri !== '/feed.xml';
}
return blockedBotRegex.test(normalizedUserAgent);
}
// rationale: README.md#asn-blocking
const blockedAsnRegex = /^(16509|14618|215125|29222|396982|1299|48090|8075|4134|23724|271315|209334|41608|213230|212238|20326|51167|53667|218785|213438|132203|201738|45102|9534|24940|197540|211619|56655|14593|18779|57043|60404|7979|62874|200651|213790|62044|136907|9009|213954|35830|43444|26548|210906)$/;
const allowedAsnBotRegex = /birchfeed\/|claude-searchbot\/|bingbot\/|mastodon\/|duckduckbot\/|humanjson-survey\//;
function isBlockedAsn(asnHeader, ua) {
return !!asnHeader && blockedAsnRegex.test(asnHeader.value) && !allowedAsnBotRegex.test(ua);
}
function createResponse(statusCode, statusDescription, contentType, body) {
return {
statusCode: statusCode,
statusDescription: statusDescription,
headers: {"content-type": {value: contentType}},
body: body
};
}
function createNotFoundResponse() {
return createResponse(404, 'Not Found', 'text/plain', 'Not Found');
}
// rationale: README.md#fake-empty-feed
const emptyAtomFeed = '<?xml version="1.0" encoding="utf-8"?>' +
'<feed xmlns="http://www.w3.org/2005/Atom"><title>Feed</title>' +
'<id>urn:uuid:00000000-0000-0000-0000-000000000000</id>' +
'<updated>2000-01-01T00:00:00Z</updated></feed>';
// rationale: README.md#fake-robots-txt
const denyAllRobotsTxt = 'User-agent: *\nDisallow: /\n';
// rationale: README.md#bad-actor-response-mapping
function createBlockedResponse(uri) {
const path = uri.toLowerCase();
// rationale: README.md#fake-robots-txt
if (path === '/robots.txt') {
return createResponse(200, 'OK', 'text/plain; charset=utf-8', denyAllRobotsTxt);
}
// rationale: README.md#fake-empty-feed
if (path === '/feed.xml') {
return createResponse(200, 'OK', 'application/atom+xml; charset=utf-8', emptyAtomFeed);
}
return createNotFoundResponse();
}
export {handler};