22

✏️ Editing: GitRemoteResolver.js

var util = require('util');
var url = require('url');
var Q = require('q');
var mout = require('mout');
var LRU = require('lru-cache');
var GitResolver = require('./GitResolver');
var cmd = require('../../util/cmd');

function GitRemoteResolver(decEndpoint, config, logger) {
    GitResolver.call(this, decEndpoint, config, logger);

if (!mout.string.startsWith(this._source, 'file://')) {
        // Trim trailing slashes
        this._source = this._source.replace(/\/+$/, '');
    }

// If the name was guessed, remove the trailing .git
    if (this._guessedName && mout.string.endsWith(this._name, '.git')) {
        this._name = this._name.slice(0, -4);
    }

// Get the remote of this source
    if (!/:\/\//.test(this._source)) {
        this._remote = url.parse('ssh://' + this._source);
    } else {
        this._remote = url.parse(this._source);
    }

this._host = this._remote.host;

// Verify whether the server supports shallow cloning
    this._shallowClone = this._supportsShallowCloning;
}

util.inherits(GitRemoteResolver, GitResolver);
mout.object.mixIn(GitRemoteResolver, GitResolver);

// -----------------

GitRemoteResolver.prototype._checkout = function() {
    var promise;
    var timer;
    var reporter;
    var that = this;
    var resolution = this._resolution;

this._logger.action(
        'checkout',
        resolution.tag || resolution.branch || resolution.commit,
        {
            resolution: resolution,
            to: this._tempDir
        }
    );

// If resolution is a commit, we need to clone the entire repo and check it out
    // Because a commit is not a named ref, there's no better solution
    if (resolution.type === 'commit') {
        promise = this._slowClone(resolution);
        // Otherwise we are checking out a named ref so we can optimize it
    } else {
        promise = this._fastClone(resolution);
    }

// Throttle the progress reporter to 1 time each sec
    reporter = mout.fn.throttle(function(data) {
        var lines;

lines = data.split(/[\r\n]+/);
        lines.forEach(function(line) {
            if (/\d{1,3}\%/.test(line)) {
                // TODO: There are some strange chars that appear once in a while (\u001b[K)
                //       Trim also those?
                that._logger.info('progress', line.trim());
            }
        });
    }, 1000);

// Start reporting progress after a few seconds
    timer = setTimeout(function() {
        promise.progress(reporter);
    }, 8000);

return (
        promise
            // Add additional proxy information to the error if necessary
            .fail(function(err) {
                that._suggestProxyWorkaround(err);
                throw err;
            })
            // Clear timer at the end
            .fin(function() {
                clearTimeout(timer);
                reporter.cancel();
            })
    );
};

GitRemoteResolver.prototype._findResolution = function(target) {
    var that = this;

// Override this function to include a meaningful message related to proxies
    // if necessary
    return GitResolver.prototype._findResolution
        .call(this, target)
        .fail(function(err) {
            that._suggestProxyWorkaround(err);
            throw err;
        });
};

// ------------------------------

GitRemoteResolver.prototype._slowClone = function(resolution) {
    return cmd('git', [
        'clone',
        this._source,
        this._tempDir,
        '--progress'
    ]).then(
        cmd.bind(cmd, 'git', ['checkout', resolution.commit], {
            cwd: this._tempDir
        })
    );
};

GitRemoteResolver.prototype._fastClone = function(resolution) {
    var branch,
        args,
        that = this;

branch = resolution.tag || resolution.branch;
    args = ['clone', this._source, '-b', branch, '--progress', '.'];

return this._shallowClone().then(function(shallowCloningSupported) {
        // If the host does not support shallow clones, we don't use --depth=1
        if (
            shallowCloningSupported &&
            !GitRemoteResolver._noShallow.get(that._host)
        ) {
            args.push('--depth', 1);
        }

return cmd('git', args, { cwd: that._tempDir }).spread(
            function(stdout, stderr) {
                // Only after 1.7.10 --branch accepts tags
                // Detect those cases and inform the user to update git otherwise it's
                // a lot slower than newer versions
                if (!/branch .+? not found/i.test(stderr)) {
                    return;
                }

that._logger.warn(
                    'old-git',
                    'It seems you are using an old version of git, it will be slower and propitious to errors!'
                );
                return cmd('git', ['checkout', resolution.commit], {
                    cwd: that._tempDir
                });
            },
            function(err) {
                // Some git servers do not support shallow clones
                // When that happens, we mark this host and try again
                if (
                    !GitRemoteResolver._noShallow.has(that._source) &&
                    err.details &&
                    /(rpc failed|shallow|--depth)/i.test(err.details)
                ) {
                    GitRemoteResolver._noShallow.set(that._host, true);
                    return that._fastClone(resolution);
                }

throw err;
            }
        );
    });
};

GitRemoteResolver.prototype._suggestProxyWorkaround = function(err) {
    if (
        (this._config.proxy || this._config.httpsProxy) &&
        mout.string.startsWith(this._source, 'git://') &&
        err.code === 'ECMDERR' &&
        err.details
    ) {
        err.details = err.details.trim();
        err.details +=
            '\n\nWhen under a proxy, you must configure git to use https:// instead of git://.';
        err.details +=
            '\nYou can configure it for every endpoint or for this specific host as follows:';
        err.details += '\ngit config --global url."https://".insteadOf git://';
        err.details +=
            '\ngit config --global url."https://' +
            this._host +
            '".insteadOf git://' +
            this._host;
        err.details +=
            'Ignore this suggestion if you already have this configured.';
    }
};

// Verifies whether the server supports shallow cloning.
// This is done according to the rules found in the following links:
// * https://github.com/dimitri/el-get/pull/1921/files
// * http://stackoverflow.com/questions/9270488/is-it-possible-to-detect-whether-a-http-git-remote-is-smart-or-dumb
//
// Summary of the rules:
// * Protocols like ssh or git always support shallow cloning
// * HTTP-based protocols can be verified by sending a HEAD or GET request to the URI (appended to the URL of the Git repo):
//       /info/refs?service=git-upload-pack
// * If the server responds with a 'Content-Type' header of 'application/x-git-upload-pack-advertisement',
//      the server supports shallow cloning ("smart server")
// * If the server responds with a different content type, the server does not support shallow cloning ("dumb server")
// * Instead of doing the HEAD or GET request using an HTTP client, we're letting Git and Curl do the heavy lifting.
//      Calling Git with the GIT_CURL_VERBOSE=2 env variable will provide the Git and Curl output, which includes
//      the content type. This has the advantage that Git will take care of using stored credentials and any additional
//      negotiation that needs to take place.
//
// The above should cover most cases, including BitBucket.
GitRemoteResolver.prototype._supportsShallowCloning = function() {
    var value = true;

// Verify that the remote could be parsed and that a protocol is set
    // This case is unlikely, but let's still cover it.
    if (this._remote == null || this._remote.protocol == null) {
        return Q.resolve(false);
    }

if (
        !this._host ||
        !this._config.shallowCloneHosts ||
        this._config.shallowCloneHosts.indexOf(this._host) === -1
    ) {
        return Q.resolve(false);
    }

// Check for protocol - the remote check for hosts supporting shallow cloning is only required for
    // HTTP or HTTPS, not for Git or SSH.
    // Also check for hosts that have been checked in a previous request and have been found to support
    // shallow cloning.
    if (
        mout.string.startsWith(this._remote.protocol, 'http') &&
        !GitRemoteResolver._canShallow.get(this._host)
    ) {
        // Provide GIT_CURL_VERBOSE=2 environment variable to capture curl output.
        // Calling ls-remote includes a call to the git-upload-pack service, which returns the content type in the response.
        var processEnv = mout.object.merge(process.env, {
            GIT_CURL_VERBOSE: '2'
        });

value = cmd('git', ['ls-remote', '--heads', this._source], {
            env: processEnv
        }).spread(
            function(stdout, stderr) {
                // Check stderr for content-type, ignore stdout
                var isSmartServer;

// If the content type is 'x-git', then the server supports shallow cloning
                isSmartServer = mout.string.contains(
                    stderr,
                    'Content-Type: application/x-git-upload-pack-advertisement'
                );

this._logger.debug(
                    'detect-smart-git',
                    'Smart Git host detected: ' + isSmartServer
                );

if (isSmartServer) {
                    // Cache this host
                    GitRemoteResolver._canShallow.set(this._host, true);
                }

return isSmartServer;
            }.bind(this)
        );
    } else {
        // One of the following cases:
        // * A non-HTTP/HTTPS protocol
        // * A host that has been checked before and that supports shallow cloning
        return Q.resolve(true);
    }

return value;
};

// ------------------------------

// Grab refs remotely
GitRemoteResolver.refs = function(source) {
    var value;

// TODO: Normalize source because of the various available protocols?
    value = this._cache.refs.get(source);
    if (value) {
        return Q.resolve(value);
    }

// Store the promise in the refs object
    value = cmd('git', ['ls-remote', '--tags', '--heads', source]).spread(
        function(stdout) {
            var refs;

refs = stdout
                .toString()
                .trim() // Trim trailing and leading spaces
                .replace(/[\t ]+/g, ' ') // Standardize spaces (some git versions make tabs, other spaces)
                .split(/[\r\n]+/); // Split lines into an array

// Update the refs with the actual refs
            this._cache.refs.set(source, refs);

return refs;
        }.bind(this)
    );

// Store the promise to be reused until it resolves
    // to a specific value
    this._cache.refs.set(source, value);

return value;
};

// Store hosts that do not support shallow clones here
GitRemoteResolver._noShallow = new LRU({ max: 50, maxAge: 5 * 60 * 1000 });

// Store hosts that support shallow clones here
GitRemoteResolver._canShallow = new LRU({ max: 50, maxAge: 5 * 60 * 1000 });

module.exports = GitRemoteResolver;

MR SKB