395 lines
11 KiB
JavaScript
395 lines
11 KiB
JavaScript
|
'use strict'
|
||
|
|
|
||
|
|
const util = require('util')
|
||
|
|
const crypto = require('crypto')
|
||
|
|
const fs = require('fs')
|
||
|
|
const Minipass = require('minipass')
|
||
|
|
const path = require('path')
|
||
|
|
const ssri = require('ssri')
|
||
|
|
const uniqueFilename = require('unique-filename')
|
||
|
|
|
||
|
|
const { disposer } = require('./util/disposer')
|
||
|
|
const contentPath = require('./content/path')
|
||
|
|
const fixOwner = require('./util/fix-owner')
|
||
|
|
const hashToSegments = require('./util/hash-to-segments')
|
||
|
|
const indexV = require('../package.json')['cache-version'].index
|
||
|
|
const moveFile = require('@npmcli/move-file')
|
||
|
|
const _rimraf = require('rimraf')
|
||
|
|
const rimraf = util.promisify(_rimraf)
|
||
|
|
rimraf.sync = _rimraf.sync
|
||
|
|
|
||
|
|
const appendFile = util.promisify(fs.appendFile)
|
||
|
|
const readFile = util.promisify(fs.readFile)
|
||
|
|
const readdir = util.promisify(fs.readdir)
|
||
|
|
const writeFile = util.promisify(fs.writeFile)
|
||
|
|
|
||
|
|
module.exports.NotFoundError = class NotFoundError extends Error {
|
||
|
|
constructor (cache, key) {
|
||
|
|
super(`No cache entry for ${key} found in ${cache}`)
|
||
|
|
this.code = 'ENOENT'
|
||
|
|
this.cache = cache
|
||
|
|
this.key = key
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.compact = compact
|
||
|
|
|
||
|
|
async function compact (cache, key, matchFn, opts = {}) {
|
||
|
|
const bucket = bucketPath(cache, key)
|
||
|
|
const entries = await bucketEntries(bucket)
|
||
|
|
const newEntries = []
|
||
|
|
// we loop backwards because the bottom-most result is the newest
|
||
|
|
// since we add new entries with appendFile
|
||
|
|
for (let i = entries.length - 1; i >= 0; --i) {
|
||
|
|
const entry = entries[i]
|
||
|
|
// a null integrity could mean either a delete was appended
|
||
|
|
// or the user has simply stored an index that does not map
|
||
|
|
// to any content. we determine if the user wants to keep the
|
||
|
|
// null integrity based on the validateEntry function passed in options.
|
||
|
|
// if the integrity is null and no validateEntry is provided, we break
|
||
|
|
// as we consider the null integrity to be a deletion of everything
|
||
|
|
// that came before it.
|
||
|
|
if (entry.integrity === null && !opts.validateEntry)
|
||
|
|
break
|
||
|
|
|
||
|
|
// if this entry is valid, and it is either the first entry or
|
||
|
|
// the newEntries array doesn't already include an entry that
|
||
|
|
// matches this one based on the provided matchFn, then we add
|
||
|
|
// it to the beginning of our list
|
||
|
|
if ((!opts.validateEntry || opts.validateEntry(entry) === true) &&
|
||
|
|
(newEntries.length === 0 ||
|
||
|
|
!newEntries.find((oldEntry) => matchFn(oldEntry, entry))))
|
||
|
|
newEntries.unshift(entry)
|
||
|
|
}
|
||
|
|
|
||
|
|
const newIndex = '\n' + newEntries.map((entry) => {
|
||
|
|
const stringified = JSON.stringify(entry)
|
||
|
|
const hash = hashEntry(stringified)
|
||
|
|
return `${hash}\t${stringified}`
|
||
|
|
}).join('\n')
|
||
|
|
|
||
|
|
const setup = async () => {
|
||
|
|
const target = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
|
||
|
|
await fixOwner.mkdirfix(cache, path.dirname(target))
|
||
|
|
return {
|
||
|
|
target,
|
||
|
|
moved: false,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const teardown = async (tmp) => {
|
||
|
|
if (!tmp.moved)
|
||
|
|
return rimraf(tmp.target)
|
||
|
|
}
|
||
|
|
|
||
|
|
const write = async (tmp) => {
|
||
|
|
await writeFile(tmp.target, newIndex, { flag: 'wx' })
|
||
|
|
await fixOwner.mkdirfix(cache, path.dirname(bucket))
|
||
|
|
// we use @npmcli/move-file directly here because we
|
||
|
|
// want to overwrite the existing file
|
||
|
|
await moveFile(tmp.target, bucket)
|
||
|
|
tmp.moved = true
|
||
|
|
try {
|
||
|
|
await fixOwner.chownr(cache, bucket)
|
||
|
|
} catch (err) {
|
||
|
|
if (err.code !== 'ENOENT')
|
||
|
|
throw err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// write the file atomically
|
||
|
|
await disposer(setup(), teardown, write)
|
||
|
|
|
||
|
|
// we reverse the list we generated such that the newest
|
||
|
|
// entries come first in order to make looping through them easier
|
||
|
|
// the true passed to formatEntry tells it to keep null
|
||
|
|
// integrity values, if they made it this far it's because
|
||
|
|
// validateEntry returned true, and as such we should return it
|
||
|
|
return newEntries.reverse().map((entry) => formatEntry(cache, entry, true))
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.insert = insert
|
||
|
|
|
||
|
|
function insert (cache, key, integrity, opts = {}) {
|
||
|
|
const { metadata, size } = opts
|
||
|
|
const bucket = bucketPath(cache, key)
|
||
|
|
const entry = {
|
||
|
|
key,
|
||
|
|
integrity: integrity && ssri.stringify(integrity),
|
||
|
|
time: Date.now(),
|
||
|
|
size,
|
||
|
|
metadata,
|
||
|
|
}
|
||
|
|
return fixOwner
|
||
|
|
.mkdirfix(cache, path.dirname(bucket))
|
||
|
|
.then(() => {
|
||
|
|
const stringified = JSON.stringify(entry)
|
||
|
|
// NOTE - Cleverness ahoy!
|
||
|
|
//
|
||
|
|
// This works because it's tremendously unlikely for an entry to corrupt
|
||
|
|
// another while still preserving the string length of the JSON in
|
||
|
|
// question. So, we just slap the length in there and verify it on read.
|
||
|
|
//
|
||
|
|
// Thanks to @isaacs for the whiteboarding session that ended up with
|
||
|
|
// this.
|
||
|
|
return appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
|
||
|
|
})
|
||
|
|
.then(() => fixOwner.chownr(cache, bucket))
|
||
|
|
.catch((err) => {
|
||
|
|
if (err.code === 'ENOENT')
|
||
|
|
return undefined
|
||
|
|
|
||
|
|
throw err
|
||
|
|
// There's a class of race conditions that happen when things get deleted
|
||
|
|
// during fixOwner, or between the two mkdirfix/chownr calls.
|
||
|
|
//
|
||
|
|
// It's perfectly fine to just not bother in those cases and lie
|
||
|
|
// that the index entry was written. Because it's a cache.
|
||
|
|
})
|
||
|
|
.then(() => {
|
||
|
|
return formatEntry(cache, entry)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.insert.sync = insertSync
|
||
|
|
|
||
|
|
function insertSync (cache, key, integrity, opts = {}) {
|
||
|
|
const { metadata, size } = opts
|
||
|
|
const bucket = bucketPath(cache, key)
|
||
|
|
const entry = {
|
||
|
|
key,
|
||
|
|
integrity: integrity && ssri.stringify(integrity),
|
||
|
|
time: Date.now(),
|
||
|
|
size,
|
||
|
|
metadata,
|
||
|
|
}
|
||
|
|
fixOwner.mkdirfix.sync(cache, path.dirname(bucket))
|
||
|
|
const stringified = JSON.stringify(entry)
|
||
|
|
fs.appendFileSync(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
|
||
|
|
try {
|
||
|
|
fixOwner.chownr.sync(cache, bucket)
|
||
|
|
} catch (err) {
|
||
|
|
if (err.code !== 'ENOENT')
|
||
|
|
throw err
|
||
|
|
}
|
||
|
|
return formatEntry(cache, entry)
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.find = find
|
||
|
|
|
||
|
|
function find (cache, key) {
|
||
|
|
const bucket = bucketPath(cache, key)
|
||
|
|
return bucketEntries(bucket)
|
||
|
|
.then((entries) => {
|
||
|
|
return entries.reduce((latest, next) => {
|
||
|
|
if (next && next.key === key)
|
||
|
|
return formatEntry(cache, next)
|
||
|
|
else
|
||
|
|
return latest
|
||
|
|
}, null)
|
||
|
|
})
|
||
|
|
.catch((err) => {
|
||
|
|
if (err.code === 'ENOENT')
|
||
|
|
return null
|
||
|
|
else
|
||
|
|
throw err
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.find.sync = findSync
|
||
|
|
|
||
|
|
function findSync (cache, key) {
|
||
|
|
const bucket = bucketPath(cache, key)
|
||
|
|
try {
|
||
|
|
return bucketEntriesSync(bucket).reduce((latest, next) => {
|
||
|
|
if (next && next.key === key)
|
||
|
|
return formatEntry(cache, next)
|
||
|
|
else
|
||
|
|
return latest
|
||
|
|
}, null)
|
||
|
|
} catch (err) {
|
||
|
|
if (err.code === 'ENOENT')
|
||
|
|
return null
|
||
|
|
else
|
||
|
|
throw err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.delete = del
|
||
|
|
|
||
|
|
function del (cache, key, opts = {}) {
|
||
|
|
if (!opts.removeFully)
|
||
|
|
return insert(cache, key, null, opts)
|
||
|
|
|
||
|
|
const bucket = bucketPath(cache, key)
|
||
|
|
return rimraf(bucket)
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.delete.sync = delSync
|
||
|
|
|
||
|
|
function delSync (cache, key, opts = {}) {
|
||
|
|
if (!opts.removeFully)
|
||
|
|
return insertSync(cache, key, null, opts)
|
||
|
|
|
||
|
|
const bucket = bucketPath(cache, key)
|
||
|
|
return rimraf.sync(bucket)
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.lsStream = lsStream
|
||
|
|
|
||
|
|
function lsStream (cache) {
|
||
|
|
const indexDir = bucketDir(cache)
|
||
|
|
const stream = new Minipass({ objectMode: true })
|
||
|
|
|
||
|
|
readdirOrEmpty(indexDir).then(buckets => Promise.all(
|
||
|
|
buckets.map(bucket => {
|
||
|
|
const bucketPath = path.join(indexDir, bucket)
|
||
|
|
return readdirOrEmpty(bucketPath).then(subbuckets => Promise.all(
|
||
|
|
subbuckets.map(subbucket => {
|
||
|
|
const subbucketPath = path.join(bucketPath, subbucket)
|
||
|
|
|
||
|
|
// "/cachename/<bucket 0xFF>/<bucket 0xFF>./*"
|
||
|
|
return readdirOrEmpty(subbucketPath).then(entries => Promise.all(
|
||
|
|
entries.map(entry => {
|
||
|
|
const entryPath = path.join(subbucketPath, entry)
|
||
|
|
return bucketEntries(entryPath).then(entries =>
|
||
|
|
// using a Map here prevents duplicate keys from
|
||
|
|
// showing up twice, I guess?
|
||
|
|
entries.reduce((acc, entry) => {
|
||
|
|
acc.set(entry.key, entry)
|
||
|
|
return acc
|
||
|
|
}, new Map())
|
||
|
|
).then(reduced => {
|
||
|
|
// reduced is a map of key => entry
|
||
|
|
for (const entry of reduced.values()) {
|
||
|
|
const formatted = formatEntry(cache, entry)
|
||
|
|
if (formatted)
|
||
|
|
stream.write(formatted)
|
||
|
|
}
|
||
|
|
}).catch(err => {
|
||
|
|
if (err.code === 'ENOENT')
|
||
|
|
return undefined
|
||
|
|
throw err
|
||
|
|
})
|
||
|
|
})
|
||
|
|
))
|
||
|
|
})
|
||
|
|
))
|
||
|
|
})
|
||
|
|
))
|
||
|
|
.then(
|
||
|
|
() => stream.end(),
|
||
|
|
err => stream.emit('error', err)
|
||
|
|
)
|
||
|
|
|
||
|
|
return stream
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.ls = ls
|
||
|
|
|
||
|
|
function ls (cache) {
|
||
|
|
return lsStream(cache).collect().then(entries =>
|
||
|
|
entries.reduce((acc, xs) => {
|
||
|
|
acc[xs.key] = xs
|
||
|
|
return acc
|
||
|
|
}, {})
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.bucketEntries = bucketEntries
|
||
|
|
|
||
|
|
function bucketEntries (bucket, filter) {
|
||
|
|
return readFile(bucket, 'utf8').then((data) => _bucketEntries(data, filter))
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.bucketEntries.sync = bucketEntriesSync
|
||
|
|
|
||
|
|
function bucketEntriesSync (bucket, filter) {
|
||
|
|
const data = fs.readFileSync(bucket, 'utf8')
|
||
|
|
return _bucketEntries(data, filter)
|
||
|
|
}
|
||
|
|
|
||
|
|
function _bucketEntries (data, filter) {
|
||
|
|
const entries = []
|
||
|
|
data.split('\n').forEach((entry) => {
|
||
|
|
if (!entry)
|
||
|
|
return
|
||
|
|
|
||
|
|
const pieces = entry.split('\t')
|
||
|
|
if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {
|
||
|
|
// Hash is no good! Corruption or malice? Doesn't matter!
|
||
|
|
// EJECT EJECT
|
||
|
|
return
|
||
|
|
}
|
||
|
|
let obj
|
||
|
|
try {
|
||
|
|
obj = JSON.parse(pieces[1])
|
||
|
|
} catch (e) {
|
||
|
|
// Entry is corrupted!
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if (obj)
|
||
|
|
entries.push(obj)
|
||
|
|
})
|
||
|
|
return entries
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.bucketDir = bucketDir
|
||
|
|
|
||
|
|
function bucketDir (cache) {
|
||
|
|
return path.join(cache, `index-v${indexV}`)
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.bucketPath = bucketPath
|
||
|
|
|
||
|
|
function bucketPath (cache, key) {
|
||
|
|
const hashed = hashKey(key)
|
||
|
|
return path.join.apply(
|
||
|
|
path,
|
||
|
|
[bucketDir(cache)].concat(hashToSegments(hashed))
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.hashKey = hashKey
|
||
|
|
|
||
|
|
function hashKey (key) {
|
||
|
|
return hash(key, 'sha256')
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports.hashEntry = hashEntry
|
||
|
|
|
||
|
|
function hashEntry (str) {
|
||
|
|
return hash(str, 'sha1')
|
||
|
|
}
|
||
|
|
|
||
|
|
function hash (str, digest) {
|
||
|
|
return crypto
|
||
|
|
.createHash(digest)
|
||
|
|
.update(str)
|
||
|
|
.digest('hex')
|
||
|
|
}
|
||
|
|
|
||
|
|
function formatEntry (cache, entry, keepAll) {
|
||
|
|
// Treat null digests as deletions. They'll shadow any previous entries.
|
||
|
|
if (!entry.integrity && !keepAll)
|
||
|
|
return null
|
||
|
|
|
||
|
|
return {
|
||
|
|
key: entry.key,
|
||
|
|
integrity: entry.integrity,
|
||
|
|
path: entry.integrity ? contentPath(cache, entry.integrity) : undefined,
|
||
|
|
size: entry.size,
|
||
|
|
time: entry.time,
|
||
|
|
metadata: entry.metadata,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function readdirOrEmpty (dir) {
|
||
|
|
return readdir(dir).catch((err) => {
|
||
|
|
if (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
||
|
|
return []
|
||
|
|
|
||
|
|
throw err
|
||
|
|
})
|
||
|
|
}
|