Skip to content

Instantly share code, notes, and snippets.

@joshuatz
Created May 12, 2019 07:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save joshuatz/4c2f321c19e503703037536751d01f29 to your computer and use it in GitHub Desktop.
Save joshuatz/4c2f321c19e503703037536751d01f29 to your computer and use it in GitHub Desktop.
File downloader that you can call with Node and pass a URL, that has options for checking file integrity after download
/**
* @author Joshua Tzucker
* @file A reusable script to download a remote file, usually to have as a dependency, that for some reason can't be fetched through NPM
*/
/**
* File should be called with arguments:
* file_downloader.js remote_file local_folder_to_save_to [newFileName="foobar.js"][forceReDownload=FALSE] [fileHash="sha256-adsfasfd"]
* fileHash should be base64 - follow rule of SRI - https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
* Example:
* node file_downloader.js https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js ./ fileHash="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo="
*/
/**
* Dependencies
*/
var fs = require('fs-extra');
var http = require('http');
var https = require('https');
var protocol = http;
/**
* Check for crypto support
*/
let crypto;
let sysHasCrypto = false;
try {
crypto = require('crypto');
sysHasCrypto = true;
}
catch (e){
//
}
/**
* Parse command line arguments
*/
let LOCAL_FOLDER = '';
let LOCAL_FILE_PATH = '';
let REMOTE_PATH = '';
let FILENAME = '';
let forceReDownload = false;
let checkHash = false;
let remoteHash = '';
process.argv.forEach((val,index,arr)=>{
// Note that first arg is automatically full path of node, second is the path of this file, and then 3rd+ are actual args passed by user
console.log(val);
if (index===2){
REMOTE_PATH = val;
if (/https/i.test(REMOTE_PATH)){
protocol = https;
}
console.log(REMOTE_PATH);
// Try to guess filename based on remote REMOTE_PATH
FILENAME = /\/([^\/]+\.[^\/]+)$/.exec(REMOTE_PATH)[1];
}
else if (index===3){
LOCAL_FOLDER = val;
}
else if (/forceReDownload=(FALSE|TRUE)/i.test(val)) {
forceReDownload = /true/i.test(val);
}
else if (/fileHash="{0,1}([^"\s]+)"{0,1}/i.test(val)) {
remoteHash = /fileHash="{0,1}([^"\s]+)"{0,1}/i.exec(val)[1];
checkHash = true;
console.log("Turning on file hash checking");
}
else if (/newFileName="{0,1}([^"\s]+)"{0,1}/i.test(val)) {
FILENAME = /newFileName="{0,1}([^"\s]+)"{0,1}/i.exec(val)[1];
}
});
LOCAL_FILE_PATH = LOCAL_FOLDER + FILENAME;
// Make sure folder exists
if(!fs.existsSync(LOCAL_FOLDER)){
console.log('Could not find ' + LOCAL_FOLDER + ' - making directory');
fs.mkdirSync(LOCAL_FOLDER);
}
downloadFile(REMOTE_PATH,LOCAL_FILE_PATH,function(res){
if (res){
console.log('File downloaded successfully!');
// Now check file integrity / hash
if (checkHash && sysHasCrypto){
// Try to extract algo from string. Assume sha256 if not specified
let allowedAlgos = ["sha256","sha384","sha512"];
let algo = "sha256";
if (/(sha\d+)-(.+)$/.test(remoteHash)){
algo = /(sha\d+)-(.+)$/.exec(remoteHash)[1];
remoteHash = /(sha\d+)-(.+)$/.exec(remoteHash)[2];
}
let computedDigestHash = '';
// Compute digest hash and compare
const fileHash = crypto.createHash(algo);
// Generate buffer from file
let localFileData = fs.readFileSync(LOCAL_FILE_PATH);
// Update hash with buffer data
fileHash.update(localFileData);
// Get base64 encoded digest of hash
computedDigestHash = fileHash.digest('base64');
if (computedDigestHash !== remoteHash){
// Uh-oh! Hash mismatch, so delete file
console.error("Hash mismatch! File deleted.");
console.warn("Was Expecting Hash: " + remoteHash);
console.warn("Computed Hash: " + computedDigestHash);
fs.removeSync(LOCAL_FILE_PATH);
}
else {
console.log("File hash matched!");
}
}
}
else {
console.error('File download failed!');
}
},!forceReDownload);
/**
* Reusable download function (reminder to self: also in zipcode project WIP)
*/
function downloadFile(remote,local,callback,OPT_useCache){
var skipDownload = false;
// Default to use cache
OPT_useCache = typeof(OPT_useCache)!=='undefined' ? OPT_useCache : true;
if (OPT_useCache && fs.existsSync(local)){
skipDownload = true;
console.log("Skipped file download; used cache instead");
callback(false);
}
if (!skipDownload){
var outputFile = fs.createWriteStream(local);
outputFile.on("open",function(){
protocol.get(remote,function(response){
response.pipe(outputFile);
response.on("end",function(){
outputFile.end();
callback(true);
});
}).on("error",function(err){
console.error("Error: " + err);
fs.unlink(local);
callback(false);
});
});
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment