Initial commit: ERP system with advance verification fixes

This commit is contained in:
System Administrator
2026-03-25 23:55:36 +07:00
commit 563ca12d76
5920 changed files with 828689 additions and 0 deletions
+40
View File
@@ -0,0 +1,40 @@
## @npmcli/agent
A pair of Agent implementations for nodejs that provide consistent keep-alives, granular timeouts, dns caching, and proxy support.
### Usage
```js
const { getAgent, HttpAgent } = require('@npmcli/agent')
const fetch = require('minipass-fetch')
const main = async () => {
// if you know what agent you need, you can create one directly
const agent = new HttpAgent(agentOptions)
// or you can use the getAgent helper, it will determine and create an Agent
// instance for you as well as reuse that agent for new requests as appropriate
const agent = getAgent('https://registry.npmjs.org/npm', agentOptions)
// minipass-fetch is just an example, this will work for any http client that
// supports node's Agents
const res = await fetch('https://registry.npmjs.org/npm', { agent })
}
main()
```
### Options
All options supported by the node Agent implementations are supported here, see [the docs](https://nodejs.org/api/http.html#new-agentoptions) for those.
Options that have been added by this module include:
- `family`: what tcp family to use, can be `4` for IPv4, `6` for IPv6 or `0` for both.
- `proxy`: a URL to a supported proxy, currently supports `HTTP CONNECT` based http/https proxies as well as socks4 and 5.
- `dns`: configuration for the built-in dns cache
- `ttl`: how long (in milliseconds) to keep cached dns entries, defaults to `5 * 60 * 100 (5 minutes)`
- `lookup`: optional function to override how dns lookups are performed, defaults to `require('dns').lookup`
- `timeouts`: a set of granular timeouts, all default to `0`
- `connection`: time between initiating connection and actually connecting
- `idle`: time between data packets (if a top level `timeout` is provided, it will be copied here)
- `response`: time between sending a request and receiving a response
- `transfer`: time between starting to receive a request and consuming the response fully
@@ -0,0 +1,206 @@
'use strict'
const net = require('net')
const tls = require('tls')
const { once } = require('events')
const timers = require('timers/promises')
const { normalizeOptions, cacheOptions } = require('./options')
const { getProxy, getProxyAgent, proxyCache } = require('./proxy.js')
const Errors = require('./errors.js')
const { Agent: AgentBase } = require('agent-base')
module.exports = class Agent extends AgentBase {
#options
#timeouts
#proxy
#noProxy
#ProxyAgent
constructor (options = {}) {
const { timeouts, proxy, noProxy, ...normalizedOptions } = normalizeOptions(options)
super(normalizedOptions)
this.#options = normalizedOptions
this.#timeouts = timeouts
if (proxy) {
this.#proxy = new URL(proxy)
this.#noProxy = noProxy
this.#ProxyAgent = getProxyAgent(proxy)
}
}
get proxy () {
return this.#proxy ? { url: this.#proxy } : {}
}
#getProxy (options) {
if (!this.#proxy) {
return
}
const proxy = getProxy(`${options.protocol}//${options.host}:${options.port}`, {
proxy: this.#proxy,
noProxy: this.#noProxy,
})
if (!proxy) {
return
}
const cacheKey = cacheOptions({
...options,
...this.#options,
timeouts: this.#timeouts,
proxy,
})
if (proxyCache.has(cacheKey)) {
return proxyCache.get(cacheKey)
}
let ProxyAgent = this.#ProxyAgent
if (Array.isArray(ProxyAgent)) {
ProxyAgent = this.isSecureEndpoint(options) ? ProxyAgent[1] : ProxyAgent[0]
}
const proxyAgent = new ProxyAgent(proxy, {
...this.#options,
socketOptions: { family: this.#options.family },
})
proxyCache.set(cacheKey, proxyAgent)
return proxyAgent
}
// takes an array of promises and races them against the connection timeout
// which will throw the necessary error if it is hit. This will return the
// result of the promise race.
async #timeoutConnection ({ promises, options, timeout }, ac = new AbortController()) {
if (timeout) {
const connectionTimeout = timers.setTimeout(timeout, null, { signal: ac.signal })
.then(() => {
throw new Errors.ConnectionTimeoutError(`${options.host}:${options.port}`)
}).catch((err) => {
if (err.name === 'AbortError') {
return
}
throw err
})
promises.push(connectionTimeout)
}
let result
try {
result = await Promise.race(promises)
ac.abort()
} catch (err) {
ac.abort()
throw err
}
return result
}
async connect (request, options) {
// if the connection does not have its own lookup function
// set, then use the one from our options
options.lookup ??= this.#options.lookup
let socket
let timeout = this.#timeouts.connection
const isSecureEndpoint = this.isSecureEndpoint(options)
const proxy = this.#getProxy(options)
if (proxy) {
// some of the proxies will wait for the socket to fully connect before
// returning so we have to await this while also racing it against the
// connection timeout.
const start = Date.now()
socket = await this.#timeoutConnection({
options,
timeout,
promises: [proxy.connect(request, options)],
})
// see how much time proxy.connect took and subtract it from
// the timeout
if (timeout) {
timeout = timeout - (Date.now() - start)
}
} else {
socket = (isSecureEndpoint ? tls : net).connect(options)
}
socket.setKeepAlive(this.keepAlive, this.keepAliveMsecs)
socket.setNoDelay(this.keepAlive)
const abortController = new AbortController()
const { signal } = abortController
const connectPromise = socket[isSecureEndpoint ? 'secureConnecting' : 'connecting']
? once(socket, isSecureEndpoint ? 'secureConnect' : 'connect', { signal })
: Promise.resolve()
await this.#timeoutConnection({
options,
timeout,
promises: [
connectPromise,
once(socket, 'error', { signal }).then((err) => {
throw err[0]
}),
],
}, abortController)
if (this.#timeouts.idle) {
socket.setTimeout(this.#timeouts.idle, () => {
socket.destroy(new Errors.IdleTimeoutError(`${options.host}:${options.port}`))
})
}
return socket
}
addRequest (request, options) {
const proxy = this.#getProxy(options)
// it would be better to call proxy.addRequest here but this causes the
// http-proxy-agent to call its super.addRequest which causes the request
// to be added to the agent twice. since we only support 3 agents
// currently (see the required agents in proxy.js) we have manually
// checked that the only public methods we need to call are called in the
// next block. this could change in the future and presumably we would get
// failing tests until we have properly called the necessary methods on
// each of our proxy agents
if (proxy?.setRequestProps) {
proxy.setRequestProps(request, options)
}
request.setHeader('connection', this.keepAlive ? 'keep-alive' : 'close')
if (this.#timeouts.response) {
let responseTimeout
request.once('finish', () => {
setTimeout(() => {
request.destroy(new Errors.ResponseTimeoutError(request, this.#proxy))
}, this.#timeouts.response)
})
request.once('response', () => {
clearTimeout(responseTimeout)
})
}
if (this.#timeouts.transfer) {
let transferTimeout
request.once('response', (res) => {
setTimeout(() => {
res.destroy(new Errors.TransferTimeoutError(request, this.#proxy))
}, this.#timeouts.transfer)
res.once('close', () => {
clearTimeout(transferTimeout)
})
})
}
return super.addRequest(request, options)
}
}
+53
View File
@@ -0,0 +1,53 @@
'use strict'
const { LRUCache } = require('lru-cache')
const dns = require('dns')
// this is a factory so that each request can have its own opts (i.e. ttl)
// while still sharing the cache across all requests
const cache = new LRUCache({ max: 50 })
const getOptions = ({
family = 0,
hints = dns.ADDRCONFIG,
all = false,
verbatim = undefined,
ttl = 5 * 60 * 1000,
lookup = dns.lookup,
}) => ({
// hints and lookup are returned since both are top level properties to (net|tls).connect
hints,
lookup: (hostname, ...args) => {
const callback = args.pop() // callback is always last arg
const lookupOptions = args[0] ?? {}
const options = {
family,
hints,
all,
verbatim,
...(typeof lookupOptions === 'number' ? { family: lookupOptions } : lookupOptions),
}
const key = JSON.stringify({ hostname, ...options })
if (cache.has(key)) {
const cached = cache.get(key)
return process.nextTick(callback, null, ...cached)
}
lookup(hostname, options, (err, ...result) => {
if (err) {
return callback(err)
}
cache.set(key, result, { ttl })
return callback(null, ...result)
})
},
})
module.exports = {
cache,
getOptions,
}
@@ -0,0 +1,61 @@
'use strict'
class InvalidProxyProtocolError extends Error {
constructor (url) {
super(`Invalid protocol \`${url.protocol}\` connecting to proxy \`${url.host}\``)
this.code = 'EINVALIDPROXY'
this.proxy = url
}
}
class ConnectionTimeoutError extends Error {
constructor (host) {
super(`Timeout connecting to host \`${host}\``)
this.code = 'ECONNECTIONTIMEOUT'
this.host = host
}
}
class IdleTimeoutError extends Error {
constructor (host) {
super(`Idle timeout reached for host \`${host}\``)
this.code = 'EIDLETIMEOUT'
this.host = host
}
}
class ResponseTimeoutError extends Error {
constructor (request, proxy) {
let msg = 'Response timeout '
if (proxy) {
msg += `from proxy \`${proxy.host}\` `
}
msg += `connecting to host \`${request.host}\``
super(msg)
this.code = 'ERESPONSETIMEOUT'
this.proxy = proxy
this.request = request
}
}
class TransferTimeoutError extends Error {
constructor (request, proxy) {
let msg = 'Transfer timeout '
if (proxy) {
msg += `from proxy \`${proxy.host}\` `
}
msg += `for \`${request.host}\``
super(msg)
this.code = 'ETRANSFERTIMEOUT'
this.proxy = proxy
this.request = request
}
}
module.exports = {
InvalidProxyProtocolError,
ConnectionTimeoutError,
IdleTimeoutError,
ResponseTimeoutError,
TransferTimeoutError,
}
@@ -0,0 +1,56 @@
'use strict'
const { LRUCache } = require('lru-cache')
const { normalizeOptions, cacheOptions } = require('./options')
const { getProxy, proxyCache } = require('./proxy.js')
const dns = require('./dns.js')
const Agent = require('./agents.js')
const agentCache = new LRUCache({ max: 20 })
const getAgent = (url, { agent, proxy, noProxy, ...options } = {}) => {
// false has meaning so this can't be a simple truthiness check
if (agent != null) {
return agent
}
url = new URL(url)
const proxyForUrl = getProxy(url, { proxy, noProxy })
const normalizedOptions = {
...normalizeOptions(options),
proxy: proxyForUrl,
}
const cacheKey = cacheOptions({
...normalizedOptions,
secureEndpoint: url.protocol === 'https:',
})
if (agentCache.has(cacheKey)) {
return agentCache.get(cacheKey)
}
const newAgent = new Agent(normalizedOptions)
agentCache.set(cacheKey, newAgent)
return newAgent
}
module.exports = {
getAgent,
Agent,
// these are exported for backwards compatability
HttpAgent: Agent,
HttpsAgent: Agent,
cache: {
proxy: proxyCache,
agent: agentCache,
dns: dns.cache,
clear: () => {
proxyCache.clear()
agentCache.clear()
dns.cache.clear()
},
},
}
@@ -0,0 +1,86 @@
'use strict'
const dns = require('./dns')
const normalizeOptions = (opts) => {
const family = parseInt(opts.family ?? '0', 10)
const keepAlive = opts.keepAlive ?? true
const normalized = {
// nodejs http agent options. these are all the defaults
// but kept here to increase the likelihood of cache hits
// https://nodejs.org/api/http.html#new-agentoptions
keepAliveMsecs: keepAlive ? 1000 : undefined,
maxSockets: opts.maxSockets ?? 15,
maxTotalSockets: Infinity,
maxFreeSockets: keepAlive ? 256 : undefined,
scheduling: 'fifo',
// then spread the rest of the options
...opts,
// we already set these to their defaults that we want
family,
keepAlive,
// our custom timeout options
timeouts: {
// the standard timeout option is mapped to our idle timeout
// and then deleted below
idle: opts.timeout ?? 0,
connection: 0,
response: 0,
transfer: 0,
...opts.timeouts,
},
// get the dns options that go at the top level of socket connection
...dns.getOptions({ family, ...opts.dns }),
}
// remove timeout since we already used it to set our own idle timeout
delete normalized.timeout
return normalized
}
const createKey = (obj) => {
let key = ''
const sorted = Object.entries(obj).sort((a, b) => a[0] - b[0])
for (let [k, v] of sorted) {
if (v == null) {
v = 'null'
} else if (v instanceof URL) {
v = v.toString()
} else if (typeof v === 'object') {
v = createKey(v)
}
key += `${k}:${v}:`
}
return key
}
const cacheOptions = ({ secureEndpoint, ...options }) => createKey({
secureEndpoint: !!secureEndpoint,
// socket connect options
family: options.family,
hints: options.hints,
localAddress: options.localAddress,
// tls specific connect options
strictSsl: secureEndpoint ? !!options.rejectUnauthorized : false,
ca: secureEndpoint ? options.ca : null,
cert: secureEndpoint ? options.cert : null,
key: secureEndpoint ? options.key : null,
// http agent options
keepAlive: options.keepAlive,
keepAliveMsecs: options.keepAliveMsecs,
maxSockets: options.maxSockets,
maxTotalSockets: options.maxTotalSockets,
maxFreeSockets: options.maxFreeSockets,
scheduling: options.scheduling,
// timeout options
timeouts: options.timeouts,
// proxy
proxy: options.proxy,
})
module.exports = {
normalizeOptions,
cacheOptions,
}
@@ -0,0 +1,88 @@
'use strict'
const { HttpProxyAgent } = require('http-proxy-agent')
const { HttpsProxyAgent } = require('https-proxy-agent')
const { SocksProxyAgent } = require('socks-proxy-agent')
const { LRUCache } = require('lru-cache')
const { InvalidProxyProtocolError } = require('./errors.js')
const PROXY_CACHE = new LRUCache({ max: 20 })
const SOCKS_PROTOCOLS = new Set(SocksProxyAgent.protocols)
const PROXY_ENV_KEYS = new Set(['https_proxy', 'http_proxy', 'proxy', 'no_proxy'])
const PROXY_ENV = Object.entries(process.env).reduce((acc, [key, value]) => {
key = key.toLowerCase()
if (PROXY_ENV_KEYS.has(key)) {
acc[key] = value
}
return acc
}, {})
const getProxyAgent = (url) => {
url = new URL(url)
const protocol = url.protocol.slice(0, -1)
if (SOCKS_PROTOCOLS.has(protocol)) {
return SocksProxyAgent
}
if (protocol === 'https' || protocol === 'http') {
return [HttpProxyAgent, HttpsProxyAgent]
}
throw new InvalidProxyProtocolError(url)
}
const isNoProxy = (url, noProxy) => {
if (typeof noProxy === 'string') {
noProxy = noProxy.split(',').map((p) => p.trim()).filter(Boolean)
}
if (!noProxy || !noProxy.length) {
return false
}
const hostSegments = url.hostname.split('.').reverse()
return noProxy.some((no) => {
const noSegments = no.split('.').filter(Boolean).reverse()
if (!noSegments.length) {
return false
}
for (let i = 0; i < noSegments.length; i++) {
if (hostSegments[i] !== noSegments[i]) {
return false
}
}
return true
})
}
const getProxy = (url, { proxy, noProxy }) => {
url = new URL(url)
if (!proxy) {
proxy = url.protocol === 'https:'
? PROXY_ENV.https_proxy
: PROXY_ENV.https_proxy || PROXY_ENV.http_proxy || PROXY_ENV.proxy
}
if (!noProxy) {
noProxy = PROXY_ENV.no_proxy
}
if (!proxy || isNoProxy(url, noProxy)) {
return null
}
return new URL(proxy)
}
module.exports = {
getProxyAgent,
getProxy,
proxyCache: PROXY_CACHE,
}
@@ -0,0 +1,60 @@
{
"name": "@npmcli/agent",
"version": "4.0.0",
"description": "the http/https agent used by the npm cli",
"main": "lib/index.js",
"scripts": {
"gencerts": "bash scripts/create-cert.sh",
"test": "tap",
"lint": "npm run eslint",
"postlint": "template-oss-check",
"template-oss-apply": "template-oss-apply --force",
"lintfix": "npm run eslint -- --fix",
"snap": "tap",
"posttest": "npm run lint",
"eslint": "eslint \"**/*.{js,cjs,ts,mjs,jsx,tsx}\""
},
"author": "GitHub Inc.",
"license": "ISC",
"bugs": {
"url": "https://github.com/npm/agent/issues"
},
"homepage": "https://github.com/npm/agent#readme",
"files": [
"bin/",
"lib/"
],
"engines": {
"node": "^20.17.0 || >=22.9.0"
},
"templateOSS": {
"//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
"version": "4.25.0",
"publish": "true"
},
"dependencies": {
"agent-base": "^7.1.0",
"http-proxy-agent": "^7.0.0",
"https-proxy-agent": "^7.0.1",
"lru-cache": "^11.2.1",
"socks-proxy-agent": "^8.0.3"
},
"devDependencies": {
"@npmcli/eslint-config": "^5.0.0",
"@npmcli/template-oss": "4.25.0",
"minipass-fetch": "^4.0.1",
"nock": "^14.0.3",
"socksv5": "^0.0.6",
"tap": "^16.3.0"
},
"repository": {
"type": "git",
"url": "git+https://github.com/npm/agent.git"
},
"tap": {
"nyc-arg": [
"--exclude",
"tap-snapshots/**"
]
}
}
+20
View File
@@ -0,0 +1,20 @@
<!-- This file is automatically added by @npmcli/template-oss. Do not edit. -->
ISC License
Copyright npm, Inc.
Permission to use, copy, modify, and/or distribute this
software for any purpose with or without fee is hereby
granted, provided that the above copyright notice and this
permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND NPM DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO
EVENT SHALL NPM BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE
USE OR PERFORMANCE OF THIS SOFTWARE.
+97
View File
@@ -0,0 +1,97 @@
# @npmcli/fs
polyfills, and extensions, of the core `fs` module.
## Features
- `fs.cp` polyfill for node < 16.7.0
- `fs.withTempDir` added
- `fs.readdirScoped` added
- `fs.moveFile` added
## `fs.withTempDir(root, fn, options) -> Promise`
### Parameters
- `root`: the directory in which to create the temporary directory
- `fn`: a function that will be called with the path to the temporary directory
- `options`
- `tmpPrefix`: a prefix to be used in the generated directory name
### Usage
The `withTempDir` function creates a temporary directory, runs the provided
function (`fn`), then removes the temporary directory and resolves or rejects
based on the result of `fn`.
```js
const fs = require('@npmcli/fs')
const os = require('os')
// this function will be called with the full path to the temporary directory
// it is called with `await` behind the scenes, so can be async if desired.
const myFunction = async (tempPath) => {
return 'done!'
}
const main = async () => {
const result = await fs.withTempDir(os.tmpdir(), myFunction)
// result === 'done!'
}
main()
```
## `fs.readdirScoped(root) -> Promise`
### Parameters
- `root`: the directory to read
### Usage
Like `fs.readdir` but handling `@org/module` dirs as if they were
a single entry.
```javascript
const { readdirScoped } = require('@npmcli/fs')
const entries = await readdirScoped('node_modules')
// entries will be something like: ['a', '@org/foo', '@org/bar']
```
## `fs.moveFile(source, dest, options) -> Promise`
A fork of [move-file](https://github.com/sindresorhus/move-file) with
support for Common JS.
### Highlights
- Promise API.
- Supports moving a file across partitions and devices.
- Optionally prevent overwriting an existing file.
- Creates non-existent destination directories for you.
- Automatically recurses when source is a directory.
### Parameters
- `source`: File, or directory, you want to move.
- `dest`: Where you want the file or directory moved.
- `options`
- `overwrite` (`boolean`, default: `true`): Overwrite existing destination file(s).
### Usage
The built-in
[`fs.rename()`](https://nodejs.org/api/fs.html#fs_fs_rename_oldpath_newpath_callback)
is just a JavaScript wrapper for the C `rename(2)` function, which doesn't
support moving files across partitions or devices. This module is what you
would have expected `fs.rename()` to be.
```js
const { moveFile } = require('@npmcli/fs');
(async () => {
await moveFile('source/unicorn.png', 'destination/unicorn.png');
console.log('The file has been moved');
})();
```
@@ -0,0 +1,20 @@
// given an input that may or may not be an object, return an object that has
// a copy of every defined property listed in 'copy'. if the input is not an
// object, assign it to the property named by 'wrap'
const getOptions = (input, { copy, wrap }) => {
const result = {}
if (input && typeof input === 'object') {
for (const prop of copy) {
if (input[prop] !== undefined) {
result[prop] = input[prop]
}
}
} else {
result[wrap] = input
}
return result
}
module.exports = getOptions
@@ -0,0 +1,9 @@
const semver = require('semver')
const satisfies = (range) => {
return semver.satisfies(process.version, range, { includePrerelease: true })
}
module.exports = {
satisfies,
}
+15
View File
@@ -0,0 +1,15 @@
(The MIT License)
Copyright (c) 2011-2017 JP Richardson
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files
(the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,129 @@
'use strict'
const { inspect } = require('util')
// adapted from node's internal/errors
// https://github.com/nodejs/node/blob/c8a04049/lib/internal/errors.js
// close copy of node's internal SystemError class.
class SystemError {
constructor (code, prefix, context) {
// XXX context.code is undefined in all constructors used in cp/polyfill
// that may be a bug copied from node, maybe the constructor should use
// `code` not `errno`? nodejs/node#41104
let message = `${prefix}: ${context.syscall} returned ` +
`${context.code} (${context.message})`
if (context.path !== undefined) {
message += ` ${context.path}`
}
if (context.dest !== undefined) {
message += ` => ${context.dest}`
}
this.code = code
Object.defineProperties(this, {
name: {
value: 'SystemError',
enumerable: false,
writable: true,
configurable: true,
},
message: {
value: message,
enumerable: false,
writable: true,
configurable: true,
},
info: {
value: context,
enumerable: true,
configurable: true,
writable: false,
},
errno: {
get () {
return context.errno
},
set (value) {
context.errno = value
},
enumerable: true,
configurable: true,
},
syscall: {
get () {
return context.syscall
},
set (value) {
context.syscall = value
},
enumerable: true,
configurable: true,
},
})
if (context.path !== undefined) {
Object.defineProperty(this, 'path', {
get () {
return context.path
},
set (value) {
context.path = value
},
enumerable: true,
configurable: true,
})
}
if (context.dest !== undefined) {
Object.defineProperty(this, 'dest', {
get () {
return context.dest
},
set (value) {
context.dest = value
},
enumerable: true,
configurable: true,
})
}
}
toString () {
return `${this.name} [${this.code}]: ${this.message}`
}
[Symbol.for('nodejs.util.inspect.custom')] (_recurseTimes, ctx) {
return inspect(this, {
...ctx,
getters: true,
customInspect: false,
})
}
}
function E (code, message) {
module.exports[code] = class NodeError extends SystemError {
constructor (ctx) {
super(code, message, ctx)
}
}
}
E('ERR_FS_CP_DIR_TO_NON_DIR', 'Cannot overwrite directory with non-directory')
E('ERR_FS_CP_EEXIST', 'Target already exists')
E('ERR_FS_CP_EINVAL', 'Invalid src or dest')
E('ERR_FS_CP_FIFO_PIPE', 'Cannot copy a FIFO pipe')
E('ERR_FS_CP_NON_DIR_TO_DIR', 'Cannot overwrite non-directory with directory')
E('ERR_FS_CP_SOCKET', 'Cannot copy a socket file')
E('ERR_FS_CP_SYMLINK_TO_SUBDIRECTORY', 'Cannot overwrite symlink in subdirectory of self')
E('ERR_FS_CP_UNKNOWN', 'Cannot copy an unknown file type')
E('ERR_FS_EISDIR', 'Path is a directory')
module.exports.ERR_INVALID_ARG_TYPE = class ERR_INVALID_ARG_TYPE extends Error {
constructor (name, expected, actual) {
super()
this.code = 'ERR_INVALID_ARG_TYPE'
this.message = `The ${name} argument must be ${expected}. Received ${typeof actual}`
}
}
@@ -0,0 +1,22 @@
const fs = require('fs/promises')
const getOptions = require('../common/get-options.js')
const node = require('../common/node.js')
const polyfill = require('./polyfill.js')
// node 16.7.0 added fs.cp
const useNative = node.satisfies('>=16.7.0')
const cp = async (src, dest, opts) => {
const options = getOptions(opts, {
copy: ['dereference', 'errorOnExist', 'filter', 'force', 'preserveTimestamps', 'recursive'],
})
// the polyfill is tested separately from this module, no need to hack
// process.version to try to trigger it just for coverage
// istanbul ignore next
return useNative
? fs.cp(src, dest, options)
: polyfill(src, dest, options)
}
module.exports = cp
@@ -0,0 +1,428 @@
// this file is a modified version of the code in node 17.2.0
// which is, in turn, a modified version of the fs-extra module on npm
// node core changes:
// - Use of the assert module has been replaced with core's error system.
// - All code related to the glob dependency has been removed.
// - Bring your own custom fs module is not currently supported.
// - Some basic code cleanup.
// changes here:
// - remove all callback related code
// - drop sync support
// - change assertions back to non-internal methods (see options.js)
// - throws ENOTDIR when rmdir gets an ENOENT for a path that exists in Windows
'use strict'
const {
ERR_FS_CP_DIR_TO_NON_DIR,
ERR_FS_CP_EEXIST,
ERR_FS_CP_EINVAL,
ERR_FS_CP_FIFO_PIPE,
ERR_FS_CP_NON_DIR_TO_DIR,
ERR_FS_CP_SOCKET,
ERR_FS_CP_SYMLINK_TO_SUBDIRECTORY,
ERR_FS_CP_UNKNOWN,
ERR_FS_EISDIR,
ERR_INVALID_ARG_TYPE,
} = require('./errors.js')
const {
constants: {
errno: {
EEXIST,
EISDIR,
EINVAL,
ENOTDIR,
},
},
} = require('os')
const {
chmod,
copyFile,
lstat,
mkdir,
readdir,
readlink,
stat,
symlink,
unlink,
utimes,
} = require('fs/promises')
const {
dirname,
isAbsolute,
join,
parse,
resolve,
sep,
toNamespacedPath,
} = require('path')
const { fileURLToPath } = require('url')
const defaultOptions = {
dereference: false,
errorOnExist: false,
filter: undefined,
force: true,
preserveTimestamps: false,
recursive: false,
}
async function cp (src, dest, opts) {
if (opts != null && typeof opts !== 'object') {
throw new ERR_INVALID_ARG_TYPE('options', ['Object'], opts)
}
return cpFn(
toNamespacedPath(getValidatedPath(src)),
toNamespacedPath(getValidatedPath(dest)),
{ ...defaultOptions, ...opts })
}
function getValidatedPath (fileURLOrPath) {
const path = fileURLOrPath != null && fileURLOrPath.href
&& fileURLOrPath.origin
? fileURLToPath(fileURLOrPath)
: fileURLOrPath
return path
}
async function cpFn (src, dest, opts) {
// Warn about using preserveTimestamps on 32-bit node
// istanbul ignore next
if (opts.preserveTimestamps && process.arch === 'ia32') {
const warning = 'Using the preserveTimestamps option in 32-bit ' +
'node is not recommended'
process.emitWarning(warning, 'TimestampPrecisionWarning')
}
const stats = await checkPaths(src, dest, opts)
const { srcStat, destStat } = stats
await checkParentPaths(src, srcStat, dest)
if (opts.filter) {
return handleFilter(checkParentDir, destStat, src, dest, opts)
}
return checkParentDir(destStat, src, dest, opts)
}
async function checkPaths (src, dest, opts) {
const { 0: srcStat, 1: destStat } = await getStats(src, dest, opts)
if (destStat) {
if (areIdentical(srcStat, destStat)) {
throw new ERR_FS_CP_EINVAL({
message: 'src and dest cannot be the same',
path: dest,
syscall: 'cp',
errno: EINVAL,
})
}
if (srcStat.isDirectory() && !destStat.isDirectory()) {
throw new ERR_FS_CP_DIR_TO_NON_DIR({
message: `cannot overwrite directory ${src} ` +
`with non-directory ${dest}`,
path: dest,
syscall: 'cp',
errno: EISDIR,
})
}
if (!srcStat.isDirectory() && destStat.isDirectory()) {
throw new ERR_FS_CP_NON_DIR_TO_DIR({
message: `cannot overwrite non-directory ${src} ` +
`with directory ${dest}`,
path: dest,
syscall: 'cp',
errno: ENOTDIR,
})
}
}
if (srcStat.isDirectory() && isSrcSubdir(src, dest)) {
throw new ERR_FS_CP_EINVAL({
message: `cannot copy ${src} to a subdirectory of self ${dest}`,
path: dest,
syscall: 'cp',
errno: EINVAL,
})
}
return { srcStat, destStat }
}
function areIdentical (srcStat, destStat) {
return destStat.ino && destStat.dev && destStat.ino === srcStat.ino &&
destStat.dev === srcStat.dev
}
function getStats (src, dest, opts) {
const statFunc = opts.dereference ?
(file) => stat(file, { bigint: true }) :
(file) => lstat(file, { bigint: true })
return Promise.all([
statFunc(src),
statFunc(dest).catch((err) => {
// istanbul ignore next: unsure how to cover.
if (err.code === 'ENOENT') {
return null
}
// istanbul ignore next: unsure how to cover.
throw err
}),
])
}
async function checkParentDir (destStat, src, dest, opts) {
const destParent = dirname(dest)
const dirExists = await pathExists(destParent)
if (dirExists) {
return getStatsForCopy(destStat, src, dest, opts)
}
await mkdir(destParent, { recursive: true })
return getStatsForCopy(destStat, src, dest, opts)
}
function pathExists (dest) {
return stat(dest).then(
() => true,
// istanbul ignore next: not sure when this would occur
(err) => (err.code === 'ENOENT' ? false : Promise.reject(err)))
}
// Recursively check if dest parent is a subdirectory of src.
// It works for all file types including symlinks since it
// checks the src and dest inodes. It starts from the deepest
// parent and stops once it reaches the src parent or the root path.
async function checkParentPaths (src, srcStat, dest) {
const srcParent = resolve(dirname(src))
const destParent = resolve(dirname(dest))
if (destParent === srcParent || destParent === parse(destParent).root) {
return
}
let destStat
try {
destStat = await stat(destParent, { bigint: true })
} catch (err) {
// istanbul ignore else: not sure when this would occur
if (err.code === 'ENOENT') {
return
}
// istanbul ignore next: not sure when this would occur
throw err
}
if (areIdentical(srcStat, destStat)) {
throw new ERR_FS_CP_EINVAL({
message: `cannot copy ${src} to a subdirectory of self ${dest}`,
path: dest,
syscall: 'cp',
errno: EINVAL,
})
}
return checkParentPaths(src, srcStat, destParent)
}
const normalizePathToArray = (path) =>
resolve(path).split(sep).filter(Boolean)
// Return true if dest is a subdir of src, otherwise false.
// It only checks the path strings.
function isSrcSubdir (src, dest) {
const srcArr = normalizePathToArray(src)
const destArr = normalizePathToArray(dest)
return srcArr.every((cur, i) => destArr[i] === cur)
}
async function handleFilter (onInclude, destStat, src, dest, opts, cb) {
const include = await opts.filter(src, dest)
if (include) {
return onInclude(destStat, src, dest, opts, cb)
}
}
function startCopy (destStat, src, dest, opts) {
if (opts.filter) {
return handleFilter(getStatsForCopy, destStat, src, dest, opts)
}
return getStatsForCopy(destStat, src, dest, opts)
}
async function getStatsForCopy (destStat, src, dest, opts) {
const statFn = opts.dereference ? stat : lstat
const srcStat = await statFn(src)
// istanbul ignore else: can't portably test FIFO
if (srcStat.isDirectory() && opts.recursive) {
return onDir(srcStat, destStat, src, dest, opts)
} else if (srcStat.isDirectory()) {
throw new ERR_FS_EISDIR({
message: `${src} is a directory (not copied)`,
path: src,
syscall: 'cp',
errno: EINVAL,
})
} else if (srcStat.isFile() ||
srcStat.isCharacterDevice() ||
srcStat.isBlockDevice()) {
return onFile(srcStat, destStat, src, dest, opts)
} else if (srcStat.isSymbolicLink()) {
return onLink(destStat, src, dest)
} else if (srcStat.isSocket()) {
throw new ERR_FS_CP_SOCKET({
message: `cannot copy a socket file: ${dest}`,
path: dest,
syscall: 'cp',
errno: EINVAL,
})
} else if (srcStat.isFIFO()) {
throw new ERR_FS_CP_FIFO_PIPE({
message: `cannot copy a FIFO pipe: ${dest}`,
path: dest,
syscall: 'cp',
errno: EINVAL,
})
}
// istanbul ignore next: should be unreachable
throw new ERR_FS_CP_UNKNOWN({
message: `cannot copy an unknown file type: ${dest}`,
path: dest,
syscall: 'cp',
errno: EINVAL,
})
}
function onFile (srcStat, destStat, src, dest, opts) {
if (!destStat) {
return _copyFile(srcStat, src, dest, opts)
}
return mayCopyFile(srcStat, src, dest, opts)
}
async function mayCopyFile (srcStat, src, dest, opts) {
if (opts.force) {
await unlink(dest)
return _copyFile(srcStat, src, dest, opts)
} else if (opts.errorOnExist) {
throw new ERR_FS_CP_EEXIST({
message: `${dest} already exists`,
path: dest,
syscall: 'cp',
errno: EEXIST,
})
}
}
async function _copyFile (srcStat, src, dest, opts) {
await copyFile(src, dest)
if (opts.preserveTimestamps) {
return handleTimestampsAndMode(srcStat.mode, src, dest)
}
return setDestMode(dest, srcStat.mode)
}
async function handleTimestampsAndMode (srcMode, src, dest) {
// Make sure the file is writable before setting the timestamp
// otherwise open fails with EPERM when invoked with 'r+'
// (through utimes call)
if (fileIsNotWritable(srcMode)) {
await makeFileWritable(dest, srcMode)
return setDestTimestampsAndMode(srcMode, src, dest)
}
return setDestTimestampsAndMode(srcMode, src, dest)
}
function fileIsNotWritable (srcMode) {
return (srcMode & 0o200) === 0
}
function makeFileWritable (dest, srcMode) {
return setDestMode(dest, srcMode | 0o200)
}
async function setDestTimestampsAndMode (srcMode, src, dest) {
await setDestTimestamps(src, dest)
return setDestMode(dest, srcMode)
}
function setDestMode (dest, srcMode) {
return chmod(dest, srcMode)
}
async function setDestTimestamps (src, dest) {
// The initial srcStat.atime cannot be trusted
// because it is modified by the read(2) system call
// (See https://nodejs.org/api/fs.html#fs_stat_time_values)
const updatedSrcStat = await stat(src)
return utimes(dest, updatedSrcStat.atime, updatedSrcStat.mtime)
}
function onDir (srcStat, destStat, src, dest, opts) {
if (!destStat) {
return mkDirAndCopy(srcStat.mode, src, dest, opts)
}
return copyDir(src, dest, opts)
}
async function mkDirAndCopy (srcMode, src, dest, opts) {
await mkdir(dest)
await copyDir(src, dest, opts)
return setDestMode(dest, srcMode)
}
async function copyDir (src, dest, opts) {
const dir = await readdir(src)
for (let i = 0; i < dir.length; i++) {
const item = dir[i]
const srcItem = join(src, item)
const destItem = join(dest, item)
const { destStat } = await checkPaths(srcItem, destItem, opts)
await startCopy(destStat, srcItem, destItem, opts)
}
}
async function onLink (destStat, src, dest) {
let resolvedSrc = await readlink(src)
if (!isAbsolute(resolvedSrc)) {
resolvedSrc = resolve(dirname(src), resolvedSrc)
}
if (!destStat) {
return symlink(resolvedSrc, dest)
}
let resolvedDest
try {
resolvedDest = await readlink(dest)
} catch (err) {
// Dest exists and is a regular file or directory,
// Windows may throw UNKNOWN error. If dest already exists,
// fs throws error anyway, so no need to guard against it here.
// istanbul ignore next: can only test on windows
if (err.code === 'EINVAL' || err.code === 'UNKNOWN') {
return symlink(resolvedSrc, dest)
}
// istanbul ignore next: should not be possible
throw err
}
if (!isAbsolute(resolvedDest)) {
resolvedDest = resolve(dirname(dest), resolvedDest)
}
if (isSrcSubdir(resolvedSrc, resolvedDest)) {
throw new ERR_FS_CP_EINVAL({
message: `cannot copy ${resolvedSrc} to a subdirectory of self ` +
`${resolvedDest}`,
path: dest,
syscall: 'cp',
errno: EINVAL,
})
}
// Do not copy if src is a subdir of dest since unlinking
// dest in this case would result in removing src contents
// and therefore a broken symlink would be created.
const srcStat = await stat(src)
if (srcStat.isDirectory() && isSrcSubdir(resolvedDest, resolvedSrc)) {
throw new ERR_FS_CP_SYMLINK_TO_SUBDIRECTORY({
message: `cannot overwrite ${resolvedDest} with ${resolvedSrc}`,
path: dest,
syscall: 'cp',
errno: EINVAL,
})
}
return copyLink(resolvedSrc, dest)
}
async function copyLink (resolvedSrc, dest) {
await unlink(dest)
return symlink(resolvedSrc, dest)
}
module.exports = cp
+13
View File
@@ -0,0 +1,13 @@
'use strict'
const cp = require('./cp/index.js')
const withTempDir = require('./with-temp-dir.js')
const readdirScoped = require('./readdir-scoped.js')
const moveFile = require('./move-file.js')
module.exports = {
cp,
withTempDir,
readdirScoped,
moveFile,
}
@@ -0,0 +1,78 @@
const { dirname, join, resolve, relative, isAbsolute } = require('path')
const fs = require('fs/promises')
const pathExists = async path => {
try {
await fs.access(path)
return true
} catch (er) {
return er.code !== 'ENOENT'
}
}
const moveFile = async (source, destination, options = {}, root = true, symlinks = []) => {
if (!source || !destination) {
throw new TypeError('`source` and `destination` file required')
}
options = {
overwrite: true,
...options,
}
if (!options.overwrite && await pathExists(destination)) {
throw new Error(`The destination file exists: ${destination}`)
}
await fs.mkdir(dirname(destination), { recursive: true })
try {
await fs.rename(source, destination)
} catch (error) {
if (error.code === 'EXDEV' || error.code === 'EPERM') {
const sourceStat = await fs.lstat(source)
if (sourceStat.isDirectory()) {
const files = await fs.readdir(source)
await Promise.all(files.map((file) =>
moveFile(join(source, file), join(destination, file), options, false, symlinks)
))
} else if (sourceStat.isSymbolicLink()) {
symlinks.push({ source, destination })
} else {
await fs.copyFile(source, destination)
}
} else {
throw error
}
}
if (root) {
await Promise.all(symlinks.map(async ({ source: symSource, destination: symDestination }) => {
let target = await fs.readlink(symSource)
// junction symlinks in windows will be absolute paths, so we need to
// make sure they point to the symlink destination
if (isAbsolute(target)) {
target = resolve(symDestination, relative(symSource, target))
}
// try to determine what the actual file is so we can create the correct
// type of symlink in windows
let targetStat = 'file'
try {
targetStat = await fs.stat(resolve(dirname(symSource), target))
if (targetStat.isDirectory()) {
targetStat = 'junction'
}
} catch {
// targetStat remains 'file'
}
await fs.symlink(
target,
symDestination,
targetStat
)
}))
await fs.rm(source, { recursive: true, force: true })
}
}
module.exports = moveFile
@@ -0,0 +1,20 @@
const { readdir } = require('fs/promises')
const { join } = require('path')
const readdirScoped = async (dir) => {
const results = []
for (const item of await readdir(dir)) {
if (item.startsWith('@')) {
for (const scopedItem of await readdir(join(dir, item))) {
results.push(join(item, scopedItem))
}
} else {
results.push(item)
}
}
return results
}
module.exports = readdirScoped
@@ -0,0 +1,39 @@
const { join, sep } = require('path')
const getOptions = require('./common/get-options.js')
const { mkdir, mkdtemp, rm } = require('fs/promises')
// create a temp directory, ensure its permissions match its parent, then call
// the supplied function passing it the path to the directory. clean up after
// the function finishes, whether it throws or not
const withTempDir = async (root, fn, opts) => {
const options = getOptions(opts, {
copy: ['tmpPrefix'],
})
// create the directory
await mkdir(root, { recursive: true })
const target = await mkdtemp(join(`${root}${sep}`, options.tmpPrefix || ''))
let err
let result
try {
result = await fn(target)
} catch (_err) {
err = _err
}
try {
await rm(target, { force: true, recursive: true })
} catch {
// ignore errors
}
if (err) {
throw err
}
return result
}
module.exports = withTempDir
@@ -0,0 +1,15 @@
#!/bin/sh
basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
case `uname` in
*CYGWIN*) basedir=`cygpath -w "$basedir"`;;
esac
if [ -x "$basedir/node" ]; then
"$basedir/node" "$basedir/../../../../semver/bin/semver.js" "$@"
ret=$?
else
node "$basedir/../../../../semver/bin/semver.js" "$@"
ret=$?
fi
exit $ret
@@ -0,0 +1,7 @@
@IF EXIST "%~dp0\node.exe" (
"%~dp0\node.exe" "%~dp0\..\..\..\..\semver\bin\semver.js" %*
) ELSE (
@SETLOCAL
@SET PATHEXT=%PATHEXT:;.JS;=;%
node "%~dp0\..\..\..\..\semver\bin\semver.js" %*
)
+54
View File
@@ -0,0 +1,54 @@
{
"name": "@npmcli/fs",
"version": "5.0.0",
"description": "filesystem utilities for the npm cli",
"main": "lib/index.js",
"files": [
"bin/",
"lib/"
],
"scripts": {
"snap": "tap",
"test": "tap",
"npmclilint": "npmcli-lint",
"lint": "npm run eslint",
"lintfix": "npm run eslint -- --fix",
"posttest": "npm run lint",
"postsnap": "npm run lintfix --",
"postlint": "template-oss-check",
"template-oss-apply": "template-oss-apply --force",
"eslint": "eslint \"**/*.{js,cjs,ts,mjs,jsx,tsx}\""
},
"repository": {
"type": "git",
"url": "git+https://github.com/npm/fs.git"
},
"keywords": [
"npm",
"oss"
],
"author": "GitHub Inc.",
"license": "ISC",
"devDependencies": {
"@npmcli/eslint-config": "^5.0.0",
"@npmcli/template-oss": "4.27.1",
"tap": "^16.0.1"
},
"dependencies": {
"semver": "^7.3.5"
},
"engines": {
"node": "^20.17.0 || >=22.9.0"
},
"templateOSS": {
"//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
"version": "4.27.1",
"publish": true
},
"tap": {
"nyc-arg": [
"--exclude",
"tap-snapshots/**"
]
}
}
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 npm
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+42
View File
@@ -0,0 +1,42 @@
# @npmcli/redact
Redact sensitive npm information from output.
## API
This will redact `npm_` prefixed tokens and UUIDs from values.
It will also replace passwords in stringified URLs.
### `redact(string)`
Redact values from a single value
```js
const { redact } = require('@npmcli/redact')
redact('https://user:pass@registry.npmjs.org/')
// https://user:***@registry.npmjs.org/
redact(`https://registry.npmjs.org/path/npm_${'a'.repeat('36')}`)
// https://registry.npmjs.org/path/npm_***
```
### `redactLog(string | string[])`
Redact values from a string or array of strings.
This method will also split all strings on `\s` and `=` and iterate over them.
```js
const { redactLog } = require('@npmcli/redact')
redactLog([
'Something --x=https://user:pass@registry.npmjs.org/ foo bar',
'--url=http://foo:bar@registry.npmjs.org',
])
// [
// 'Something --x=https://user:***@registry.npmjs.org/ foo bar',
// '--url=http://foo:***@registry.npmjs.org/',
// ]
```
@@ -0,0 +1,71 @@
const { serializeError } = require('./error')
const deepMap = (input, handler = v => v, path = ['$'], seen = new Set([input])) => {
// this is in an effort to maintain bole's error logging behavior
if (path.join('.') === '$' && input instanceof Error) {
return deepMap({ err: serializeError(input) }, handler, path, seen)
}
if (input instanceof Error) {
return deepMap(serializeError(input), handler, path, seen)
}
// allows for non-node js environments, sush as workers
if (typeof Buffer !== 'undefined' && input instanceof Buffer) {
return `[unable to log instanceof buffer]`
}
if (input instanceof Uint8Array) {
return `[unable to log instanceof Uint8Array]`
}
if (Array.isArray(input)) {
const result = []
for (let i = 0; i < input.length; i++) {
const element = input[i]
const elementPath = [...path, i]
if (element instanceof Object) {
if (!seen.has(element)) { // avoid getting stuck in circular reference
seen.add(element)
result.push(deepMap(handler(element, elementPath), handler, elementPath, seen))
}
} else {
result.push(handler(element, elementPath))
}
}
return result
}
if (input === null) {
return null
} else if (typeof input === 'object' || typeof input === 'function') {
const result = {}
for (const propertyName of Object.getOwnPropertyNames(input)) {
// skip logging internal properties
if (propertyName.startsWith('_')) {
continue
}
try {
const property = input[propertyName]
const propertyPath = [...path, propertyName]
if (property instanceof Object) {
if (!seen.has(property)) { // avoid getting stuck in circular reference
seen.add(property)
result[propertyName] = deepMap(
handler(property, propertyPath), handler, propertyPath, seen
)
}
} else {
result[propertyName] = handler(property, propertyPath)
}
} catch (err) {
// a getter may throw an error
result[propertyName] = `[error getting value: ${err.message}]`
}
}
return result
}
return handler(input, path)
}
module.exports = { deepMap }
@@ -0,0 +1,28 @@
/** takes an error object and serializes it to a plan object */
function serializeError (input) {
if (!(input instanceof Error)) {
if (typeof input === 'string') {
const error = new Error(`attempted to serialize a non-error, string String, "${input}"`)
return serializeError(error)
}
const error = new Error(`attempted to serialize a non-error, ${typeof input} ${input?.constructor?.name}`)
return serializeError(error)
}
// different error objects store status code differently
// AxiosError uses `status`, other services use `statusCode`
const statusCode = input.statusCode ?? input.status
// CAUTION: what we serialize here gets add to the size of logs
return {
errorType: input.errorType ?? input.constructor.name,
...(input.message ? { message: input.message } : {}),
...(input.stack ? { stack: input.stack } : {}),
// think of this as error code
...(input.code ? { code: input.code } : {}),
// think of this as http status code
...(statusCode ? { statusCode } : {}),
}
}
module.exports = {
serializeError,
}
@@ -0,0 +1,44 @@
const matchers = require('./matchers')
const { redactUrlPassword } = require('./utils')
const REPLACE = '***'
const redact = (value) => {
if (typeof value !== 'string' || !value) {
return value
}
return redactUrlPassword(value, REPLACE)
.replace(matchers.NPM_SECRET.pattern, `npm_${REPLACE}`)
.replace(matchers.UUID.pattern, REPLACE)
}
// split on \s|= similar to how nopt parses options
const splitAndRedact = (str) => {
// stateful regex, don't move out of this scope
const splitChars = /[\s=]/g
let match = null
let result = ''
let index = 0
while (match = splitChars.exec(str)) {
result += redact(str.slice(index, match.index)) + match[0]
index = splitChars.lastIndex
}
return result + redact(str.slice(index))
}
// replaces auth info in an array of arguments or in a strings
const redactLog = (arg) => {
if (typeof arg === 'string') {
return splitAndRedact(arg)
} else if (Array.isArray(arg)) {
return arg.map((a) => typeof a === 'string' ? splitAndRedact(a) : a)
}
return arg
}
module.exports = {
redact,
redactLog,
}
@@ -0,0 +1,88 @@
const TYPE_REGEX = 'regex'
const TYPE_URL = 'url'
const TYPE_PATH = 'path'
const NPM_SECRET = {
type: TYPE_REGEX,
pattern: /\b(npms?_)[a-zA-Z0-9]{36,48}\b/gi,
replacement: `[REDACTED_NPM_SECRET]`,
}
const AUTH_HEADER = {
type: TYPE_REGEX,
pattern: /\b(Basic\s+|Bearer\s+)[\w+=\-.]+\b/gi,
replacement: `[REDACTED_AUTH_HEADER]`,
}
const JSON_WEB_TOKEN = {
type: TYPE_REGEX,
pattern: /\b[A-Za-z0-9-_]{10,}(?!\.\d+\.)\.[A-Za-z0-9-_]{3,}\.[A-Za-z0-9-_]{20,}\b/gi,
replacement: `[REDACTED_JSON_WEB_TOKEN]`,
}
const UUID = {
type: TYPE_REGEX,
pattern: /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi,
replacement: `[REDACTED_UUID]`,
}
const URL_MATCHER = {
type: TYPE_REGEX,
pattern: /(?:https?|ftp):\/\/[^\s/"$.?#].[^\s"]*/gi,
replacement: '[REDACTED_URL]',
}
const DEEP_HEADER_AUTHORIZATION = {
type: TYPE_PATH,
predicate: ({ path }) => path.endsWith('.headers.authorization'),
replacement: '[REDACTED_HEADER_AUTHORIZATION]',
}
const DEEP_HEADER_SET_COOKIE = {
type: TYPE_PATH,
predicate: ({ path }) => path.endsWith('.headers.set-cookie'),
replacement: '[REDACTED_HEADER_SET_COOKIE]',
}
const DEEP_HEADER_COOKIE = {
type: TYPE_PATH,
predicate: ({ path }) => path.endsWith('.headers.cookie'),
replacement: '[REDACTED_HEADER_COOKIE]',
}
const REWRITE_REQUEST = {
type: TYPE_PATH,
predicate: ({ path }) => path.endsWith('.request'),
replacement: (input) => ({
method: input?.method,
path: input?.path,
headers: input?.headers,
url: input?.url,
}),
}
const REWRITE_RESPONSE = {
type: TYPE_PATH,
predicate: ({ path }) => path.endsWith('.response'),
replacement: (input) => ({
data: input?.data,
status: input?.status,
headers: input?.headers,
}),
}
module.exports = {
TYPE_REGEX,
TYPE_URL,
TYPE_PATH,
NPM_SECRET,
AUTH_HEADER,
JSON_WEB_TOKEN,
UUID,
URL_MATCHER,
DEEP_HEADER_AUTHORIZATION,
DEEP_HEADER_SET_COOKIE,
DEEP_HEADER_COOKIE,
REWRITE_REQUEST,
REWRITE_RESPONSE,
}
@@ -0,0 +1,59 @@
const {
AUTH_HEADER,
JSON_WEB_TOKEN,
NPM_SECRET,
DEEP_HEADER_AUTHORIZATION,
DEEP_HEADER_SET_COOKIE,
REWRITE_REQUEST,
REWRITE_RESPONSE,
DEEP_HEADER_COOKIE,
} = require('./matchers')
const {
redactUrlMatcher,
redactUrlPasswordMatcher,
redactMatchers,
} = require('./utils')
const { serializeError } = require('./error')
const { deepMap } = require('./deep-map')
const _redact = redactMatchers(
NPM_SECRET,
AUTH_HEADER,
JSON_WEB_TOKEN,
DEEP_HEADER_AUTHORIZATION,
DEEP_HEADER_SET_COOKIE,
DEEP_HEADER_COOKIE,
REWRITE_REQUEST,
REWRITE_RESPONSE,
redactUrlMatcher(
redactUrlPasswordMatcher()
)
)
const redact = (input) => deepMap(input, (value, path) => _redact(value, { path }))
/** takes an error returns new error keeping some custom properties */
function redactError (input) {
const { message, ...data } = serializeError(input)
const output = new Error(redact(message))
return Object.assign(output, redact(data))
}
/** runs a function within try / catch and throws error wrapped in redactError */
function redactThrow (func) {
if (typeof func !== 'function') {
throw new Error('redactThrow expects a function')
}
return async (...args) => {
try {
return await func(...args)
} catch (error) {
throw redactError(error)
}
}
}
module.exports = { redact, redactError, redactThrow }
@@ -0,0 +1,202 @@
const {
URL_MATCHER,
TYPE_URL,
TYPE_REGEX,
TYPE_PATH,
} = require('./matchers')
/**
* creates a string of asterisks,
* this forces a minimum asterisk for security purposes
*/
const asterisk = (length = 0) => {
length = typeof length === 'string' ? length.length : length
if (length < 8) {
return '*'.repeat(8)
}
return '*'.repeat(length)
}
/**
* escapes all special regex chars
* @see https://stackoverflow.com/a/9310752
* @see https://github.com/tc39/proposal-regex-escaping
*/
const escapeRegExp = (text) => {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, `\\$&`)
}
/**
* provieds a regex "or" of the url versions of a string
*/
const urlEncodeRegexGroup = (value) => {
const decoded = decodeURIComponent(value)
const encoded = encodeURIComponent(value)
const union = [...new Set([encoded, decoded, value])].map(escapeRegExp).join('|')
return union
}
/**
* a tagged template literal that returns a regex ensures all variables are excaped
*/
const urlEncodeRegexTag = (strings, ...values) => {
let pattern = ''
for (let i = 0; i < values.length; i++) {
pattern += strings[i] + `(${urlEncodeRegexGroup(values[i])})`
}
pattern += strings[strings.length - 1]
return new RegExp(pattern)
}
/**
* creates a matcher for redacting url hostname
*/
const redactUrlHostnameMatcher = ({ hostname, replacement } = {}) => ({
type: TYPE_URL,
predicate: ({ url }) => url.hostname === hostname,
pattern: ({ url }) => {
return urlEncodeRegexTag`(^${url.protocol}//${url.username}:.+@)?${url.hostname}`
},
replacement: `$1${replacement || asterisk()}`,
})
/**
* creates a matcher for redacting url search / query parameter values
*/
const redactUrlSearchParamsMatcher = ({ param, replacement } = {}) => ({
type: TYPE_URL,
predicate: ({ url }) => url.searchParams.has(param),
pattern: ({ url }) => urlEncodeRegexTag`(${param}=)${url.searchParams.get(param)}`,
replacement: `$1${replacement || asterisk()}`,
})
/** creates a matcher for redacting the url password */
const redactUrlPasswordMatcher = ({ replacement } = {}) => ({
type: TYPE_URL,
predicate: ({ url }) => url.password,
pattern: ({ url }) => urlEncodeRegexTag`(^${url.protocol}//${url.username}:)${url.password}`,
replacement: `$1${replacement || asterisk()}`,
})
const redactUrlReplacement = (...matchers) => (subValue) => {
try {
const url = new URL(subValue)
return redactMatchers(...matchers)(subValue, { url })
} catch (err) {
return subValue
}
}
/**
* creates a matcher / submatcher for urls, this function allows you to first
* collect all urls within a larger string and then pass those urls to a
* submatcher
*
* @example
* console.log("this will first match all urls, then pass those urls to the password patcher")
* redactMatchers(redactUrlMatcher(redactUrlPasswordMatcher()))
*
* @example
* console.log(
* "this will assume you are passing in a string that is a url, and will redact the password"
* )
* redactMatchers(redactUrlPasswordMatcher())
*
*/
const redactUrlMatcher = (...matchers) => {
return {
...URL_MATCHER,
replacement: redactUrlReplacement(...matchers),
}
}
const matcherFunctions = {
[TYPE_REGEX]: (matcher) => (value) => {
if (typeof value === 'string') {
value = value.replace(matcher.pattern, matcher.replacement)
}
return value
},
[TYPE_URL]: (matcher) => (value, ctx) => {
if (typeof value === 'string') {
try {
const url = ctx?.url || new URL(value)
const { predicate, pattern } = matcher
const predicateValue = predicate({ url })
if (predicateValue) {
value = value.replace(pattern({ url }), matcher.replacement)
}
} catch (_e) {
return value
}
}
return value
},
[TYPE_PATH]: (matcher) => (value, ctx) => {
const rawPath = ctx?.path
const path = rawPath.join('.').toLowerCase()
const { predicate, replacement } = matcher
const replace = typeof replacement === 'function' ? replacement : () => replacement
const shouldRun = predicate({ rawPath, path })
if (shouldRun) {
value = replace(value, { rawPath, path })
}
return value
},
}
/** converts a matcher to a function */
const redactMatcher = (matcher) => {
return matcherFunctions[matcher.type](matcher)
}
/** converts a series of matchers to a function */
const redactMatchers = (...matchers) => (value, ctx) => {
const flatMatchers = matchers.flat()
return flatMatchers.reduce((result, matcher) => {
const fn = (typeof matcher === 'function') ? matcher : redactMatcher(matcher)
return fn(result, ctx)
}, value)
}
/**
* replacement handler, keeping $1 (if it exists) and replacing the
* rest of the string with asterisks, maintaining string length
*/
const redactDynamicReplacement = () => (value, start) => {
if (typeof start === 'number') {
return asterisk(value)
}
return start + asterisk(value.substring(start.length).length)
}
/**
* replacement handler, keeping $1 (if it exists) and replacing the
* rest of the string with a fixed number of asterisks
*/
const redactFixedReplacement = (length) => (_value, start) => {
if (typeof start === 'number') {
return asterisk(length)
}
return start + asterisk(length)
}
const redactUrlPassword = (value, replacement) => {
return redactMatchers(redactUrlPasswordMatcher({ replacement }))(value)
}
module.exports = {
asterisk,
escapeRegExp,
urlEncodeRegexGroup,
urlEncodeRegexTag,
redactUrlHostnameMatcher,
redactUrlSearchParamsMatcher,
redactUrlPasswordMatcher,
redactUrlMatcher,
redactUrlReplacement,
redactDynamicReplacement,
redactFixedReplacement,
redactMatchers,
redactUrlPassword,
}
@@ -0,0 +1,52 @@
{
"name": "@npmcli/redact",
"version": "4.0.0",
"description": "Redact sensitive npm information from output",
"main": "lib/index.js",
"exports": {
".": "./lib/index.js",
"./server": "./lib/server.js",
"./package.json": "./package.json"
},
"scripts": {
"test": "tap",
"lint": "npm run eslint",
"postlint": "template-oss-check",
"template-oss-apply": "template-oss-apply --force",
"lintfix": "npm run eslint -- --fix",
"snap": "tap",
"posttest": "npm run lint",
"eslint": "eslint \"**/*.{js,cjs,ts,mjs,jsx,tsx}\""
},
"keywords": [],
"author": "GitHub Inc.",
"license": "ISC",
"files": [
"bin/",
"lib/"
],
"repository": {
"type": "git",
"url": "git+https://github.com/npm/redact.git"
},
"templateOSS": {
"//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
"version": "4.27.1",
"publish": true
},
"tap": {
"nyc-arg": [
"--exclude",
"tap-snapshots/**"
],
"timeout": 120
},
"devDependencies": {
"@npmcli/eslint-config": "^5.0.0",
"@npmcli/template-oss": "4.27.1",
"tap": "^16.3.10"
},
"engines": {
"node": "^20.17.0 || >=22.9.0"
}
}