diff --git a/docs/ky.md b/docs/ky.md index 6400976..fbee50d 100644 --- a/docs/ky.md +++ b/docs/ky.md @@ -2,6 +2,8 @@ [ky](https://github.com/sindresorhus/ky) is a small HTTP client built on `fetch`. This package wires ky to a custom `fetch` implemented with [node-fetch](node-fetch.md) and `ProxyHeadersAgent`, so CONNECT proxy headers work the same way as in the node-fetch adapter. +ky 2.2+ throws `The fetch option must resolve with a Response` for a node-fetch `Response`. The adapter therefore returns a global `Response` with the same `proxyHeaders` Map attached. `node-fetch` is still required. + ## Getting Started ### Prerequisites @@ -25,7 +27,7 @@ const api = await createProxyKy({ const response = await api('https://httpbin.org/ip'); const data = await response.json(); -// CONNECT response headers (ProxyResponse) +// CONNECT response headers console.log(response.proxyHeaders.get('x-proxymesh-ip')); ``` @@ -71,4 +73,4 @@ const res = await api.get('v1/status').json(); ## Accessing Proxy Headers -Responses are [`ProxyResponse`](node-fetch.md) instances: use `response.proxyHeaders.get('x-proxymesh-ip')` (and the usual `response.json()`, `response.text()`, etc.). Proxy CONNECT failures use the same errors as the underlying fetch stack; see [Core API](core-api.md) for `ConnectError` on the agent. +Use `response.proxyHeaders.get('x-proxymesh-ip')` (and the usual `response.json()`, `response.text()`, etc.). `response` is a global `Response`; `proxyHeaders` is an own property and is not copied by `response.clone()`. Proxy CONNECT failures use the same errors as the underlying fetch stack; see [Core API](core-api.md) for `ConnectError` on the agent. diff --git a/lib/ky-proxy.js b/lib/ky-proxy.js index bb2ffcb..c05ae94 100644 --- a/lib/ky-proxy.js +++ b/lib/ky-proxy.js @@ -2,10 +2,89 @@ * ky extension for proxy header support. * * Uses a custom `fetch` backed by node-fetch + ProxyHeadersAgent. + * ky 2.2+ requires `fetch` to resolve with a global `Response` + * (`instanceof Response` or `[object Response]`). node-fetch's Response + * and the ProxyResponse wrapper fail that check, so ky 2 receives a + * native Response with `proxyHeaders` attached. `clone()` does not copy + * that own property. */ +import { Readable } from 'node:stream'; import { createProxyFetch } from './node-fetch-proxy.js'; +const NULL_BODY_STATUS = new Set([101, 204, 205, 304]); +const DROP_HEADERS = new Set([ + 'connection', + 'keep-alive', + 'proxy-connection', + 'transfer-encoding', + 'upgrade', + 'trailer', + 'te', + // node-fetch already decompresses when Content-Encoding is set, but leaves + // the header in place. Copying it makes the native Response decode twice. + 'content-encoding', + 'content-length', +]); + +/** + * @param {import('./core/proxy-response.js').ProxyResponse} proxyResponse + * @returns {Response} + */ +function toNativeResponse(proxyResponse) { + if (typeof Response !== 'undefined' && proxyResponse instanceof Response) { + if (!proxyResponse.proxyHeaders) { + proxyResponse.proxyHeaders = new Map(); + } + return proxyResponse; + } + + const status = Number(proxyResponse.status ?? 200); + const nullBody = NULL_BODY_STATUS.has(status); + let body = null; + if (!nullBody && proxyResponse.body) { + const stream = proxyResponse.body; + body = typeof stream.getReader === 'function' + ? stream + : Readable.toWeb(stream); + } + + const native = new Response(body, { + status, + statusText: proxyResponse.statusText || '', + headers: copyHeaders(proxyResponse.headers), + }); + native.proxyHeaders = proxyResponse.proxyHeaders || new Map(); + return native; +} + +/** + * @param {{ raw?: () => Record, entries?: () => Iterable<[string, string]> }|undefined} source + * @returns {Headers} + */ +function copyHeaders(source) { + const headers = new Headers(); + if (!source) { + return headers; + } + const raw = typeof source.raw === 'function' ? source.raw() : null; + const pairs = raw + ? Object.entries(raw).flatMap(([name, values]) => + [].concat(values).map((value) => [name, value])) + : [...source.entries()]; + for (const [name, value] of pairs) { + if (value == null || DROP_HEADERS.has(String(name).toLowerCase())) { + continue; + } + try { + headers.append(name, String(value)); + } catch { + // The Fetch implementation rejects some header names. + } + } + return headers; +} + /** * Create a ky instance with proxy header support. * @@ -13,6 +92,7 @@ import { createProxyFetch } from './node-fetch-proxy.js'; * @param {string} options.proxy - Proxy URL * @param {Object} [options.proxyHeaders] - Headers to send on CONNECT * @param {Function} [options.onProxyConnect] - CONNECT callback + * @param {Object} [options.proxyTlsOptions] - TLS options for an https:// proxy * @param {Object} [options.kyOptions] - Options passed to ky.create() * @returns {Promise} */ @@ -30,7 +110,8 @@ export async function createProxyKy(options) { throw new Error('ky is required. Install it with: npm install ky'); } - const fetch = createProxyFetch({ proxy, proxyHeaders, onProxyConnect, proxyTlsOptions }); + const fetchImpl = createProxyFetch({ proxy, proxyHeaders, onProxyConnect, proxyTlsOptions }); + const fetch = async (input, init) => toNativeResponse(await fetchImpl(input, init)); return ky.create({ ...kyOptions, diff --git a/package-lock.json b/package-lock.json index a297f01..be15875 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "node-fetch": "^3.0.0", "superagent": "^8.0.0 || ^9.0.0 || ^10.0.0", "typed-rest-client": "^2.0.0", - "undici": "^5.0.0 || ^6.0.0 || ^7.0.0", + "undici": "^8.0.2", "wretch": "^2.0.0 || ^3.0.0" }, "peerDependenciesMeta": { @@ -2376,9 +2376,9 @@ } }, "node_modules/ky": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ky/-/ky-2.1.0.tgz", - "integrity": "sha512-nIKwelw+7qVqKOlX4Eht6GXfEXlDHcdIjyLWV3s119kYt5s+70ayTnYS+0fRNmtgGRRAfqdeWQbh8YizqkS0ng==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ky/-/ky-2.2.0.tgz", + "integrity": "sha512-4+B63kyZt+Y8snQiUGydK1vgY9xYQJZEh9K/nIAIEts+gYJzy1EiAw3BbK8xpWRuWUpN70SNCMjqRGTm75xxIA==", "dev": true, "license": "MIT", "engines": { @@ -3360,9 +3360,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "8.12.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.2.tgz", - "integrity": "sha512-XtuDWVdqTLr3NAxm2ly9/hU6kUsJOgGh1RASzMBbsw6aWCOaQzHdW+QQDfdERjfJ2V5vL5pRPa4RetO9UfivDg==", + "version": "8.12.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.3.tgz", + "integrity": "sha512-+fdndUTwBP86cVjihTp21hZHo7d6oj1c/47XPA3lL8mM0XHs+C4EkcjmgRsFnFk2f/UONGDy9ItBItmXsLOQHw==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index f379b4f..ff014c8 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ ], "scripts": { "release:pr": "node scripts/create-release-pr.mjs", - "test:unit": "node --test test/test_connect_security.js", + "test:unit": "node --test test/test_connect_security.js test/test_ky_response.js", "test": "node test/test_proxy_headers.js core axios node-fetch got undici superagent ky wretch make-fetch-happen needle typed-rest-client", "test:run": "node run_tests.js", "test:verbose": "node test/test_proxy_headers.js -v", diff --git a/test/test_ky_response.js b/test/test_ky_response.js new file mode 100644 index 0000000..10608e3 --- /dev/null +++ b/test/test_ky_response.js @@ -0,0 +1,142 @@ +#!/usr/bin/env node +/** + * ky 2.2 requires fetch to resolve with a global Response. + * Does not require a live PROXY_URL. + */ + +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import net from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import tls from 'node:tls'; +import { gzipSync } from 'node:zlib'; + +import { createProxyKy } from '../lib/ky-proxy.js'; + +function listen(server) { + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); +} + +function makeSelfSignedCert() { + const dir = mkdtempSync(join(tmpdir(), 'jph-ky-')); + const keyPath = join(dir, 'key.pem'); + const certPath = join(dir, 'cert.pem'); + execFileSync('openssl', [ + 'req', '-x509', '-newkey', 'rsa:2048', + '-keyout', keyPath, '-out', certPath, + '-days', '1', '-nodes', '-subj', '/CN=127.0.0.1', + ], { stdio: 'pipe' }); + return { + key: readFileSync(keyPath), + cert: readFileSync(certPath), + cleanup: () => rmSync(dir, { recursive: true, force: true }), + }; +} + +/** + * CONNECT proxy that terminates TLS and returns a small JSON origin response. + * @param {{ cert: Buffer, key: Buffer, gzip?: boolean }} options + */ +function createJsonProxy({ cert, key, gzip = false }) { + const connects = []; + const payload = JSON.stringify({ ok: true }); + const rawBody = gzip ? gzipSync(payload) : Buffer.from(payload); + const extraHeaders = gzip + ? `Content-Encoding: gzip\r\nContent-Length: ${rawBody.length}\r\n` + : `Content-Length: ${rawBody.length}\r\n`; + + const server = net.createServer((sock) => { + let buf = Buffer.alloc(0); + let handedOff = false; + sock.on('data', (d) => { + if (handedOff) return; + buf = Buffer.concat([buf, d]); + if (buf.indexOf('\r\n\r\n') === -1) return; + handedOff = true; + connects.push(buf.toString('utf8')); + sock.write( + 'HTTP/1.1 200 Connection Established\r\n' + + 'X-ProxyMesh-IP: 203.0.113.22\r\n' + + '\r\n', + ); + const tlsSock = new tls.TLSSocket(sock, { isServer: true, cert, key }); + tlsSock.on('secure', () => { + let httpBuf = Buffer.alloc(0); + tlsSock.on('data', (chunk) => { + httpBuf = Buffer.concat([httpBuf, chunk]); + if (httpBuf.indexOf('\r\n\r\n') === -1) return; + const head = Buffer.from( + 'HTTP/1.1 200 OK\r\n' + + 'Content-Type: application/json\r\n' + + extraHeaders + + 'Connection: close\r\n' + + '\r\n', + ); + tlsSock.end(Buffer.concat([head, rawBody])); + }); + }); + tlsSock.on('error', () => {}); + }); + sock.on('error', () => {}); + }); + server.connects = connects; + return server; +} + +async function withProxy(gzip, fn) { + const previous = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + const { cert, key, cleanup } = makeSelfSignedCert(); + const proxy = createJsonProxy({ cert, key, gzip }); + await listen(proxy); + try { + await fn(proxy); + } finally { + proxy.close(); + cleanup(); + if (previous === undefined) { + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + } else { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = previous; + } + } +} + +test('ky 2 fetch resolves to a global Response that keeps proxyHeaders', async () => { + await withProxy(false, async (proxy) => { + const { port } = proxy.address(); + const api = await createProxyKy({ + proxy: `http://127.0.0.1:${port}`, + proxyHeaders: { 'X-ProxyMesh-Country': 'US' }, + }); + const response = await api('https://127.0.0.1/ip'); + assert.equal(response instanceof Response, true); + assert.equal(Object.prototype.toString.call(response), '[object Response]'); + assert.equal(response.proxyHeaders.get('x-proxymesh-ip'), '203.0.113.22'); + assert.match(proxy.connects[0], /X-ProxyMesh-Country: US/); + + const cloned = response.clone(); + assert.equal(cloned.proxyHeaders, undefined); + + assert.deepEqual(await response.json(), { ok: true }); + }); +}); + +test('ky 2 reads a gzip origin body node-fetch already decompressed', async () => { + await withProxy(true, async (proxy) => { + const { port } = proxy.address(); + const api = await createProxyKy({ + proxy: `http://127.0.0.1:${port}`, + }); + const response = await api('https://127.0.0.1/ip'); + assert.equal(response instanceof Response, true); + assert.equal(response.headers.get('content-encoding'), null); + assert.deepEqual(await response.json(), { ok: true }); + assert.equal(response.proxyHeaders.get('x-proxymesh-ip'), '203.0.113.22'); + }); +});