22

✏️ Editing: ResolveCache.js

var fs = require('../util/fs');
var path = require('path');
var mout = require('mout');
var Q = require('q');
var mkdirp = require('mkdirp');
var rimraf = require('../util/rimraf');
var LRU = require('lru-cache');
var lockFile = require('lockfile');
var md5 = require('md5-hex');
var semver = require('../util/semver');
var readJson = require('../util/readJson');
var copy = require('../util/copy');

function ResolveCache(config) {
    // TODO: Make some config entries, such as:
    //       - Max MB
    //       - Max versions per source
    //       - Max MB per source
    //       - etc..
    this._config = config;
    this._dir = this._config.storage.packages;
    this._lockDir = this._config.storage.packages;

mkdirp.sync(this._lockDir);

// Cache is stored/retrieved statically to ensure singularity
    // among instances
    this._cache = this.constructor._cache.get(this._dir);
    if (!this._cache) {
        this._cache = new LRU({
            max: 100,
            maxAge: 60 * 5 * 1000 // 5 minutes
        });
        this.constructor._cache.set(this._dir, this._cache);
    }

// Ensure dir is created
    mkdirp.sync(this._dir);
}

// -----------------

ResolveCache.prototype.retrieve = function(source, target) {
    var sourceId = md5(source);
    var dir = path.join(this._dir, sourceId);
    var that = this;

target = target || '*';

return this._getVersions(sourceId)
        .spread(function(versions) {
            var suitable;

// If target is a semver, find a suitable version
            if (semver.validRange(target)) {
                suitable = semver.maxSatisfying(versions, target, true);

if (suitable) {
                    return suitable;
                }
            }

// If target is '*' check if there's a cached '_wildcard'
            if (target === '*') {
                return mout.array.find(versions, function(version) {
                    return version === '_wildcard';
                });
            }

// Otherwise check if there's an exact match
            return mout.array.find(versions, function(version) {
                return version === target;
            });
        })
        .then(function(version) {
            var canonicalDir;

if (!version) {
                return [];
            }

// Resolve with canonical dir and package meta
            canonicalDir = path.join(dir, encodeURIComponent(version));
            return that._readPkgMeta(canonicalDir).then(
                function(pkgMeta) {
                    return [canonicalDir, pkgMeta];
                },
                function() {
                    // If there was an error, invalidate the in-memory cache,
                    // delete the cached package and try again
                    that._cache.del(sourceId);

return Q.nfcall(rimraf, canonicalDir).then(function() {
                        return that.retrieve(source, target);
                    });
                }
            );
        });
};

ResolveCache.prototype.store = function(canonicalDir, pkgMeta) {
    var sourceId;
    var release;
    var dir;
    var pkgLock;
    var promise;
    var that = this;

promise = pkgMeta ? Q.resolve(pkgMeta) : this._readPkgMeta(canonicalDir);

return promise
        .then(function(pkgMeta) {
            sourceId = md5(pkgMeta._source);
            release = that._getPkgRelease(pkgMeta);
            dir = path.join(that._dir, sourceId, release);
            pkgLock = path.join(
                that._lockDir,
                sourceId + '-' + release + '.lock'
            );

// Check if destination directory exists to prevent issuing lock at all times
            return Q.nfcall(fs.stat, dir)
                .fail(function(err) {
                    var lockParams = { wait: 250, retries: 25, stale: 60000 };
                    return Q.nfcall(lockFile.lock, pkgLock, lockParams)
                        .then(function() {
                            // Ensure other process didn't start copying files before lock was created
                            return Q.nfcall(fs.stat, dir).fail(function(err) {
                                // If stat fails, it is expected to return ENOENT
                                if (err.code !== 'ENOENT') {
                                    throw err;
                                }

// Create missing directory and copy files there
                                return Q.nfcall(mkdirp, path.dirname(dir)).then(
                                    function() {
                                        return Q.nfcall(
                                            fs.rename,
                                            canonicalDir,
                                            dir
                                        ).fail(function(err) {
                                            // If error is EXDEV it means that we are trying to rename
                                            // across different drives, so we copy and remove it instead
                                            if (err.code !== 'EXDEV') {
                                                throw err;
                                            }

return copy.copyDir(
                                                canonicalDir,
                                                dir
                                            );
                                        });
                                    }
                                );
                            });
                        })
                        .finally(function() {
                            lockFile.unlockSync(pkgLock);
                        });
                })
                .finally(function() {
                    // Ensure no tmp dir is left on disk.
                    return Q.nfcall(rimraf, canonicalDir);
                });
        })
        .then(function() {
            var versions = that._cache.get(sourceId);

// Add it to the in memory cache
            // and sort the versions afterwards
            if (versions && versions.indexOf(release) === -1) {
                versions.push(release);
                that._sortVersions(versions);
            }

// Resolve with the final location
            return dir;
        });
};

ResolveCache.prototype.eliminate = function(pkgMeta) {
    var sourceId = md5(pkgMeta._source);
    var release = this._getPkgRelease(pkgMeta);
    var dir = path.join(this._dir, sourceId, release);
    var that = this;

return Q.nfcall(rimraf, dir).then(function() {
        var versions = that._cache.get(sourceId) || [];
        mout.array.remove(versions, release);

// If this was the last package in the cache,
        // delete the parent folder (source)
        // For extra security, check against the file system
        // if this was really the last package
        if (!versions.length) {
            that._cache.del(sourceId);

return that._getVersions(sourceId).spread(function(versions) {
                if (!versions.length) {
                    // Do not keep in-memory cache if it's completely
                    // empty
                    that._cache.del(sourceId);

return Q.nfcall(rimraf, path.dirname(dir));
                }
            });
        }
    });
};

ResolveCache.prototype.clear = function() {
    return Q.nfcall(rimraf, this._dir)
        .then(
            function() {
                return Q.nfcall(fs.mkdir, this._dir);
            }.bind(this)
        )
        .then(
            function() {
                this._cache.reset();
            }.bind(this)
        );
};

ResolveCache.prototype.reset = function() {
    this._cache.reset();
    return this;
};

ResolveCache.prototype.versions = function(source) {
    var sourceId = md5(source);

return this._getVersions(sourceId).spread(function(versions) {
        return versions.filter(function(version) {
            return semver.valid(version);
        });
    });
};

ResolveCache.prototype.list = function() {
    var promises;
    var dirs = [];
    var that = this;

// Get the list of directories
    return (
        Q.nfcall(fs.readdir, this._dir)
            .then(function(sourceIds) {
                promises = sourceIds.map(function(sourceId) {
                    return Q.nfcall(
                        fs.readdir,
                        path.join(that._dir, sourceId)
                    ).then(
                        function(versions) {
                            versions.forEach(function(version) {
                                var dir = path.join(
                                    that._dir,
                                    sourceId,
                                    version
                                );
                                dirs.push(dir);
                            });
                        },
                        function(err) {
                            // Ignore lurking files, e.g.: .DS_Store if the user
                            // has navigated throughout the cache
                            if (err.code === 'ENOTDIR' && err.path) {
                                return Q.nfcall(rimraf, err.path);
                            }

throw err;
                        }
                    );
                });

return Q.all(promises);
            })
            // Read every package meta
            .then(function() {
                promises = dirs.map(function(dir) {
                    return that._readPkgMeta(dir).then(
                        function(pkgMeta) {
                            return {
                                canonicalDir: dir,
                                pkgMeta: pkgMeta
                            };
                        },
                        function() {
                            // If it fails to read, invalidate the in memory
                            // cache for the source and delete the entry directory
                            var sourceId = path.basename(path.dirname(dir));
                            that._cache.del(sourceId);

return Q.nfcall(rimraf, dir);
                        }
                    );
                });

return Q.all(promises);
            })
            // Sort by name ASC & release ASC
            .then(function(entries) {
                // Ignore falsy entries due to errors reading
                // package metas
                entries = entries.filter(function(entry) {
                    return !!entry;
                });

return entries.sort(function(entry1, entry2) {
                    var pkgMeta1 = entry1.pkgMeta;
                    var pkgMeta2 = entry2.pkgMeta;
                    var comp = pkgMeta1.name.localeCompare(pkgMeta2.name);

// Sort by name
                    if (comp) {
                        return comp;
                    }

// Sort by version
                    if (pkgMeta1.version && pkgMeta2.version) {
                        return semver.compare(
                            pkgMeta1.version,
                            pkgMeta2.version
                        );
                    }
                    if (pkgMeta1.version) {
                        return -1;
                    }
                    if (pkgMeta2.version) {
                        return 1;
                    }

// Sort by target
                    return pkgMeta1._target.localeCompare(pkgMeta2._target);
                });
            })
    );
};

// ------------------------

ResolveCache.clearRuntimeCache = function() {
    // Note that _cache refers to the static _cache variable
    // that holds other caches per dir!
    // Do not confuse it with the instance cache

// Clear cache of each directory
    this._cache.forEach(function(cache) {
        cache.reset();
    });

// Clear root cache
    this._cache.reset();
};

// ------------------------

ResolveCache.prototype._getPkgRelease = function(pkgMeta) {
    var release =
        pkgMeta.version ||
        (pkgMeta._target === '*' ? '_wildcard' : pkgMeta._target);

// Encode some dangerous chars such as / and \
    release = encodeURIComponent(release);

return release;
};

ResolveCache.prototype._readPkgMeta = function(dir) {
    var filename = path.join(dir, '.bower.json');

return readJson(filename).spread(function(json) {
        return json;
    });
};

ResolveCache.prototype._getVersions = function(sourceId) {
    var dir;
    var versions = this._cache.get(sourceId);
    var that = this;

if (versions) {
        return Q.resolve([versions, true]);
    }

dir = path.join(this._dir, sourceId);
    return Q.nfcall(fs.readdir, dir).then(
        function(versions) {
            // Sort and cache in memory
            that._sortVersions(versions);
            versions = versions.map(decodeURIComponent);
            that._cache.set(sourceId, versions);
            return [versions, false];
        },
        function(err) {
            // If the directory does not exists, resolve
            // as an empty array
            if (err.code === 'ENOENT') {
                versions = [];
                that._cache.set(sourceId, versions);
                return [versions, false];
            }

throw err;
        }
    );
};

ResolveCache.prototype._sortVersions = function(versions) {
    // Sort DESC
    versions.sort(function(version1, version2) {
        var validSemver1 = semver.valid(version1);
        var validSemver2 = semver.valid(version2);

// If both are semvers, compare them
        if (validSemver1 && validSemver2) {
            return semver.rcompare(version1, version2);
        }

// If one of them are semvers, give higher priority
        if (validSemver1) {
            return -1;
        }
        if (validSemver2) {
            return 1;
        }

// Otherwise they are considered equal
        return 0;
    });
};

// ------------------------

ResolveCache._cache = new LRU({
    max: 5,
    maxAge: 60 * 30 * 1000 // 30 minutes
});

module.exports = ResolveCache;

MR SKB