From a0a5c588941566d5f3b3979f6385595fae7c2c74 Mon Sep 17 00:00:00 2001
From: Artur Adib <arturadib@gmail.com>
Date: Mon, 26 Mar 2012 16:33:00 -0400
Subject: [PATCH 1/6] Upgrading ShellJS, introducing 'makeref'

---
 external/shelljs/ChangeLog    |  22 +++
 external/shelljs/README.md    |  61 ++++---
 external/shelljs/package.json |   2 +-
 external/shelljs/shell.js     | 295 ++++++++++++++++++++++++----------
 make.js                       |  47 ++++--
 5 files changed, 304 insertions(+), 123 deletions(-)
 create mode 100644 external/shelljs/ChangeLog

diff --git a/external/shelljs/ChangeLog b/external/shelljs/ChangeLog
new file mode 100644
index 000000000..23ca5d0e0
--- /dev/null
+++ b/external/shelljs/ChangeLog
@@ -0,0 +1,22 @@
+2012.03.22, Version 0.0.4
+
+* ls() and find() return arrays instead of hashes (Artur Adib)
+* exec({silent:...}) overrides global silent() state (Artur Adib)
+
+
+2012.03.21, Version 0.0.3
+
+* Wildcard bug fix (Artur Adib)
+* execSync() now uses dummy file I/O op to reduce CPU usage (Artur Adib)
+* Minor fixes
+
+
+2012.03.15, Version 0.0.2
+
+* New methods: find(), test() (Artur Adib)
+* Deprecated non-Unix methods: exists(), verbose()
+
+
+2012.03.03, Version 0.0.2pre1
+
+* First public release
diff --git a/external/shelljs/README.md b/external/shelljs/README.md
index 82f53533b..9f5386efe 100644
--- a/external/shelljs/README.md
+++ b/external/shelljs/README.md
@@ -1,6 +1,7 @@
 # ShellJS - Unix shell commands for Node.js [![Build Status](https://secure.travis-ci.org/arturadib/shelljs.png)](http://travis-ci.org/arturadib/shelljs)
 
-_This project is young and experimental. Use at your own risk._
++ _This project is young and experimental. Use at your own risk._
++ _Major API change as of v0.0.4: `ls()` and `find()` now return arrays._
 
 ShellJS is a **portable** (Windows included) implementation of Unix shell commands on top of the Node.js API. You can use it to eliminate your shell script's dependency on Unix while still keeping its familiar and powerful commands.
 
@@ -18,11 +19,11 @@ cp('-R', 'stuff/*', 'out/Release');
 
 // Replace macros in each .js file
 cd('lib');
-for (file in ls('*.js')) {
+ls('*.js').forEach(function(file) {
   sed('-i', 'BUILD_VERSION', 'v0.1.2', file);
   sed('-i', /.*REMOVE_THIS_LINE.*\n/, '', file);
   sed('-i', /.*REPLACE_LINE_WITH_MACRO.*\n/, cat('macro.js'), file);
-}
+});
 cd('..');
 
 // Run external tool synchronously
@@ -73,11 +74,11 @@ target.docs = function() {
   cd(__dirname);
   mkdir('docs');
   cd('lib');
-  for (file in ls('*.js')) {
+  ls('*.js').forEach(function(file){
     var text = grep('//@', file); // extract special comments
     text.replace('//@', ''); // remove comment tags
     text.to('docs/my_docs.md');
-  }
+  });
 }
 ```
 
@@ -128,9 +129,7 @@ ls('-R', '/users/me', '/tmp');
 ls('-R', ['/users/me', '/tmp']); // same as above
 ```
 
-Returns list of files in the given path, or in current directory if no path provided.
-For convenient iteration via `for (file in ls())`, the format returned is a hash object:
-`{ 'file1':null, 'dir1/file2':null, ...}`.
+Returns array of files in the given path, or in current directory if no path provided.
 
 #### find(path [,path ...])
 #### find(path_array)
@@ -139,18 +138,12 @@ Examples:
 ```javascript
 find('src', 'lib');
 find(['src', 'lib']); // same as above
-for (file in find('.')) {
-if (!file.match(/\.js$/))
-continue;
-// all files at this point end in '.js'
-}
+find('.').filter(function(file) { return file.match(/\.js$/); })
 ```
 
-Returns list of all files (however deep) in the given paths. For convenient iteration 
-via `for (file in find(...))`, the format returned is a hash object:
-`{ 'file1':null, 'dir1/file2':null, ...}`.
+Returns array of all files (however deep) in the given paths.
 
-The main difference with respect to `ls('-R', path)` is that the resulting file names 
+The main difference from `ls('-R', path)` is that the resulting file names 
 include the base directories, e.g. `lib/resources/file1` instead of just `file1`.
 
 #### cp('[options ,] source [,source ...], dest')
@@ -339,16 +332,34 @@ arguments `(code, output)`.
 Searches and returns string containing a writeable, platform-dependent temporary directory.
 Follows Python's [tempfile algorithm](http://docs.python.org/library/tempfile.html#tempfile.tempdir).
 
-#### exists(path [, path ...])
-#### exists(path_array)
-Returns true if all the given paths exist.
-
 #### error()
 Tests if error occurred in the last command. Returns `null` if no error occurred,
 otherwise returns string explaining the error
 
-#### verbose()
-Enables all output (default)
+#### silent([state])
+Example:
 
-#### silent()
-Suppresses all output, except for explict `echo()` calls
+```javascript
+var silentState = silent();
+silent(true);
+/* ... */
+silent(silentState); // restore old silent state
+```
+
+Suppresses all output if `state = true`. Returns state if no arguments given.
+
+## Deprecated
+
+
+#### exists(path [, path ...])
+#### exists(path_array)
+
+_This function is being deprecated. Use `test()` instead._
+
+Returns true if all the given paths exist.
+
+#### verbose()
+
+_This function is being deprecated. Use `silent(false) instead.`_
+
+Enables all output (default)
diff --git a/external/shelljs/package.json b/external/shelljs/package.json
index 9499680e9..99c0f5e64 100644
--- a/external/shelljs/package.json
+++ b/external/shelljs/package.json
@@ -1,5 +1,5 @@
 { "name": "shelljs"
-, "version": "0.0.2pre1"
+, "version": "0.0.5pre2"
 , "author": "Artur Adib <aadib@mozilla.com>"
 , "description": "Portable Unix shell commands for Node.js"
 , "keywords": ["unix", "shell", "makefile", "make", "jake", "synchronous"]
diff --git a/external/shelljs/shell.js b/external/shelljs/shell.js
index 95ddaa067..9c1181fb6 100644
--- a/external/shelljs/shell.js
+++ b/external/shelljs/shell.js
@@ -74,9 +74,7 @@ exports.pwd = wrap('pwd', _pwd);
 //@ ls('-R', ['/users/me', '/tmp']); // same as above
 //@ ```
 //@
-//@ Returns list of files in the given path, or in current directory if no path provided.
-//@ For convenient iteration via `for (file in ls())`, the format returned is a hash object:
-//@ `{ 'file1':null, 'dir1/file2':null, ...}`.
+//@ Returns array of files in the given path, or in current directory if no path provided.
 function _ls(options, paths) {
   options = parseOptions(options, {
     'R': 'recursive',
@@ -90,24 +88,30 @@ function _ls(options, paths) {
   else if (typeof paths === 'string')
     paths = [].slice.call(arguments, 1);
 
-  var hash = {};
+  var list = [];
 
-  function pushHash(file, query) {
+  // Conditionally pushes file to list - returns true if pushed, false otherwise
+  // (e.g. prevents hidden files to be included unless explicitly told so)
+  function pushFile(file, query) {
     // hidden file?
     if (path.basename(file)[0] === '.') {
       // not explicitly asking for hidden files?
       if (!options.all && !(path.basename(query)[0] === '.' && path.basename(query).length > 1))
-        return;
+        return false;
     }
 
-    hash[file] = null;
+    if (platform === 'win')
+      file = file.replace(/\\/g, '/');
+
+    list.push(file);
+    return true;
   }
 
   paths.forEach(function(p) {
     if (fs.existsSync(p)) {
       // Simple file?
       if (fs.statSync(p).isFile()) {
-        pushHash(p, p);
+        pushFile(p, p);
         return; // continue
       }
       
@@ -115,14 +119,17 @@ function _ls(options, paths) {
       if (fs.statSync(p).isDirectory()) {
         // Iterate over p contents
         fs.readdirSync(p).forEach(function(file) {
-          pushHash(file, p);
+          if (!pushFile(file, p))
+            return;
 
-          // Recursive
-          var oldDir = _pwd();
-          _cd('', p);
-          if (fs.statSync(file).isDirectory() && options.recursive)
-            hash = extend(hash, _ls('-R', file+'/*'));
-          _cd('', oldDir);
+          // Recursive?
+          if (options.recursive) {
+            var oldDir = _pwd();
+            _cd('', p);
+            if (fs.statSync(file).isDirectory())
+              list = list.concat(_ls('-R'+(options.all?'a':''), file+'/*'));
+            _cd('', oldDir);
+          }
         });
         return; // continue
       }
@@ -137,17 +144,20 @@ function _ls(options, paths) {
       // Escape special regular expression chars
       var regexp = basename.replace(/(\^|\$|\(|\)|\<|\>|\[|\]|\{|\}|\.|\+|\?)/g, '\\$1');
       // Translates wildcard into regex
-      regexp = '^' + regexp.replace(/\*/g, '.*');
+      regexp = '^' + regexp.replace(/\*/g, '.*') + '$';
       // Iterate over directory contents
       fs.readdirSync(dirname).forEach(function(file) {
         if (file.match(new RegExp(regexp))) {
-          pushHash(path.normalize(dirname+'/'+file), basename);
+          if (!pushFile(path.normalize(dirname+'/'+file), basename))
+            return;
 
-          // Recursive
-          var pp = dirname + '/' + file;
-          if (fs.statSync(pp).isDirectory() && options.recursive)
-            hash = extend(hash, _ls('-R', pp+'/*'));
-        }
+          // Recursive?
+          if (options.recursive) {
+            var pp = dirname + '/' + file;
+            if (fs.statSync(pp).isDirectory())
+              list = list.concat(_ls('-R'+(options.all?'a':''), pp+'/*'));
+          } // recursive
+        } // if file matches
       }); // forEach
       return;
     }
@@ -155,7 +165,7 @@ function _ls(options, paths) {
     error('no such file or directory: ' + p, true);
   });
 
-  return hash;
+  return list;
 };
 exports.ls = wrap('ls', _ls);
 
@@ -168,16 +178,10 @@ exports.ls = wrap('ls', _ls);
 //@ ```javascript
 //@ find('src', 'lib');
 //@ find(['src', 'lib']); // same as above
-//@ for (file in find('.')) {
-//@   if (!file.match(/\.js$/))
-//@     continue;
-//@   // all files at this point end in '.js'
-//@ }
+//@ find('.').filter(function(file) { return file.match(/\.js$/); });
 //@ ```
 //@
-//@ Returns list of all files (however deep) in the given paths. For convenient iteration 
-//@ via `for (file in find(...))`, the format returned is a hash object:
-//@ `{ 'file1':null, 'dir1/file2':null, ...}`.
+//@ Returns array of all files (however deep) in the given paths.
 //@
 //@ The main difference from `ls('-R', path)` is that the resulting file names 
 //@ include the base directories, e.g. `lib/resources/file1` instead of just `file1`.
@@ -189,21 +193,28 @@ function _find(options, paths) {
   else if (typeof paths === 'string')
     paths = [].slice.call(arguments, 1);
 
-  var hash = {};
+  var list = [];
+
+  function pushFile(file) {
+    if (platform === 'win')
+      file = file.replace(/\\/g, '/');
+    list.push(file);
+  }
 
   // why not simply do ls('-R', paths)? because the output wouldn't give the base dirs
   // to get the base dir in the output, we need instead ls('-R', 'dir/*') for every directory
 
-  paths.forEach(function(file){
-    hash[file] = null;
+  paths.forEach(function(file) {
+    pushFile(file);
 
     if (fs.statSync(file).isDirectory()) {
-      for (subfile in _ls('-Ra', file+'/*'))
-        hash[subfile] = null;
+      _ls('-Ra', file+'/*').forEach(function(subfile) {
+        pushFile(subfile);
+      });
     }
   });
 
-  return hash;
+  return list;
 }
 exports.find = wrap('find', _find);
 
@@ -347,9 +358,20 @@ function _rm(options, files) {
 
     // Remove simple file
     if (fs.statSync(file).isFile()) {
-      fs.unlinkSync(file);
+
+      // Do not check for file writing permissions
+      if (options.force) {
+        _unlinkSync(file);
+        return;
+      }
+            
+      if (isWriteable(file))
+        _unlinkSync(file);
+      else
+        error('permission denied: '+file, true);
+
       return;
-    }
+    } // simple file
 
     // Path is an existing directory, but no -r flag given
     if (fs.statSync(file).isDirectory() && !options.recursive) {
@@ -359,7 +381,7 @@ function _rm(options, files) {
 
     // Recursively remove existing directory
     if (fs.statSync(file).isDirectory() && options.recursive) {
-      rmdirSyncRecursive(file);
+      rmdirSyncRecursive(file, options.force);
     }
   }); // forEach(file)
 }; // rm
@@ -582,7 +604,11 @@ function _to(options, file) {
   if (!fs.existsSync( path.dirname(file) ))
       error('no such file or directory: ' + path.dirname(file));
 
-  fs.writeFileSync(file, this.toString(), 'utf8');
+  try {
+    fs.writeFileSync(file, this.toString(), 'utf8');
+  } catch(e) {
+    error('could not write to file (code '+e.code+'): '+file, true);
+  }
 };
 // In the future, when Proxies are default, we can add methods like `.to()` to primitive strings. 
 // For now, this is a dummy function to bookmark places we need such strings
@@ -793,7 +819,7 @@ function _exec(command, options, callback) {
   }
 
   options = extend({
-    silent: false,
+    silent: state.silent,
     async: false
   }, options);
 
@@ -822,11 +848,52 @@ exports.exec = wrap('exec', _exec, {notUnix:true});
 //@ Follows Python's [tempfile algorithm](http://docs.python.org/library/tempfile.html#tempfile.tempdir).
 exports.tempdir = wrap('tempdir', tempDir);
 
+
+//@
+//@ #### error()
+//@ Tests if error occurred in the last command. Returns `null` if no error occurred,
+//@ otherwise returns string explaining the error
+exports.error = function() {
+  return state.error;
+}
+
+//@
+//@ #### silent([state])
+//@ Example:
+//@
+//@ ```javascript
+//@ var silentState = silent();
+//@ silent(true);
+//@ /* ... */
+//@ silent(silentState); // restore old silent state
+//@ ```
+//@
+//@ Suppresses all output if `state = true`. Returns state if no arguments given.
+exports.silent = function(_state) {
+  if (typeof _state !== 'boolean')
+    return state.silent;
+  
+  state.silent = _state;
+}
+
+
+//@
+//@ ## Deprecated
+//@
+
+
+
+
 //@
 //@ #### exists(path [, path ...])
 //@ #### exists(path_array)
+//@
+//@ _This function is being deprecated. Use `test()` instead._
+//@
 //@ Returns true if all the given paths exist.
 function _exists(options, paths) {
+  deprecate('exists', 'Use test() instead.');
+
   if (!paths)
     error('no paths given');
 
@@ -844,32 +911,19 @@ function _exists(options, paths) {
 };
 exports.exists = wrap('exists', _exists);
 
-//@
-//@ #### error()
-//@ Tests if error occurred in the last command. Returns `null` if no error occurred,
-//@ otherwise returns string explaining the error
-exports.error = function() {
-  return state.error;
-}
 
 //@
 //@ #### verbose()
+//@
+//@ _This function is being deprecated. Use `silent(false) instead.`_
+//@
 //@ Enables all output (default)
 exports.verbose = function() {
+  deprecate('verbose', 'Use silent(false) instead.');
+
   state.silent = false;
 }
 
-//@
-//@ #### silent()
-//@ Suppresses all output, except for explict `echo()` calls
-exports.silent = function() {
-  state.silent = true;
-}
-
-
-
-
-
 
 
 
@@ -889,6 +943,10 @@ function log() {
     console.log.apply(this, arguments);
 }
 
+function deprecate(what, msg) {
+  console.log('*** ShellJS.'+what+': This function is deprecated.', msg);
+}
+
 function write(msg) {
   if (!state.silent)
     process.stdout.write(msg);
@@ -984,10 +1042,22 @@ function copyFileSync(srcFile, destFile) {
 
   var BUF_LENGTH = 64*1024,
       buf = new Buffer(BUF_LENGTH),
-      fdr = fs.openSync(srcFile, 'r'),
-      fdw = fs.openSync(destFile, 'w'),
       bytesRead = BUF_LENGTH,
-      pos = 0;
+      pos = 0,
+      fdr = null,
+      fdw = null;
+
+  try {
+    fdr = fs.openSync(srcFile, 'r');
+  } catch(e) {
+    error('copyFileSync: could not read src file ('+srcFile+')');
+  }
+
+  try {
+    fdw = fs.openSync(destFile, 'w');
+  } catch(e) {
+    error('copyFileSync: could not write to dest file (code='+e.code+'):'+destFile);
+  }
 
   while (bytesRead === BUF_LENGTH) {
     bytesRead = fs.readSync(fdr, buf, 0, BUF_LENGTH, pos);
@@ -1050,28 +1120,42 @@ function cpdirSyncRecursive(sourceDir, destDir, opts) {
 //
 // Licensed under the MIT License
 // http://www.opensource.org/licenses/mit-license.php
-function rmdirSyncRecursive(dir) {
+function rmdirSyncRecursive(dir, force) {
   var files;
 
   files = fs.readdirSync(dir);
 
   // Loop through and delete everything in the sub-tree after checking it
   for(var i = 0; i < files.length; i++) {
-    var currFile = fs.lstatSync(dir + "/" + files[i]);
+    var file = dir + "/" + files[i],
+        currFile = fs.lstatSync(file);
 
-    if(currFile.isDirectory()) // Recursive function back to the beginning
-      rmdirSyncRecursive(dir + "/" + files[i]);
+    if(currFile.isDirectory()) { // Recursive function back to the beginning
+      rmdirSyncRecursive(file, force);
+    }
 
-    else if(currFile.isSymbolicLink()) // Unlink symlinks
-      fs.unlinkSync(dir + "/" + files[i]);
+    else if(currFile.isSymbolicLink()) { // Unlink symlinks
+      if (force || isWriteable(file))
+        _unlinkSync(file);
+    }
 
     else // Assume it's a file - perhaps a try/catch belongs here?
-      fs.unlinkSync(dir + "/" + files[i]);
+      if (force || isWriteable(file))
+        _unlinkSync(file);
   }
 
   // Now that we know everything in the sub-tree has been deleted, we can delete the main directory. 
   // Huzzah for the shopkeep.
-  return fs.rmdirSync(dir);
+
+  var result;
+  try {
+    result = fs.rmdirSync(dir);
+  } catch(e) {
+    if (e.code === 'ENOTEMPTY')
+      error('directory not empty: ' + dir, true);
+  }
+
+  return result;
 }; // rmdirSyncRecursive
 
 // Recursively creates 'dir'
@@ -1118,7 +1202,7 @@ function writeableDir(dir) {
   var testFile = dir+'/'+randomFileName();
   try {
     fs.writeFileSync(testFile, ' ');
-    fs.unlinkSync(testFile);
+    _unlinkSync(testFile);
     return dir;
   } catch (e) {
     return false;
@@ -1150,7 +1234,8 @@ function tempDir() {
 
 // Wrapper around exec() to enable echoing output to console in real time
 function execAsync(cmd, opts, callback) {
-  var output = '';
+  var output = '',
+      silent = 'silent' in opts ? opts.silent : state.silent;
   
   var c = child.exec(cmd, {env: process.env}, function(err) {
     if (callback) 
@@ -1159,13 +1244,13 @@ function execAsync(cmd, opts, callback) {
 
   c.stdout.on('data', function(data) {
     output += data;
-    if (!opts.silent)
+    if (!silent)
       write(data);
   });
 
   c.stderr.on('data', function(data) {
     output += data;
-    if (!opts.silent)
+    if (!silent)
       write(data);
   });
 }
@@ -1178,16 +1263,17 @@ function execAsync(cmd, opts, callback) {
 function execSync(cmd, opts) {
   var stdoutFile = path.resolve(tempDir()+'/'+randomFileName()),
       codeFile = path.resolve(tempDir()+'/'+randomFileName()),
-      scriptFile = path.resolve(tempDir()+'/'+randomFileName());
+      scriptFile = path.resolve(tempDir()+'/'+randomFileName()),
+      sleepFile = path.resolve(tempDir()+'/'+randomFileName());
 
   var options = extend({
-    silent: false
+    silent: state.silent
   }, opts);
 
   var previousStdoutContent = '';
   // Echoes stdout changes from running process, if not silent
   function updateStdout() {
-    if (state.silent || options.silent || !fs.existsSync(stdoutFile))
+    if (options.silent || !fs.existsSync(stdoutFile))
       return;
 
     var stdoutContent = fs.readFileSync(stdoutFile, 'utf8');
@@ -1214,9 +1300,9 @@ function execSync(cmd, opts) {
       fs.writeFileSync('"+escape(codeFile)+"', err ? err.code.toString() : '0'); \
     });";
 
-  if (fs.existsSync(scriptFile)) fs.unlinkSync(scriptFile);
-  if (fs.existsSync(stdoutFile)) fs.unlinkSync(stdoutFile);
-  if (fs.existsSync(codeFile)) fs.unlinkSync(codeFile);
+  if (fs.existsSync(scriptFile)) _unlinkSync(scriptFile);
+  if (fs.existsSync(stdoutFile)) _unlinkSync(stdoutFile);
+  if (fs.existsSync(codeFile)) _unlinkSync(codeFile);
 
   fs.writeFileSync(scriptFile, script);
   child.exec('node '+scriptFile, { 
@@ -1225,8 +1311,11 @@ function execSync(cmd, opts) {
   });
 
   // The wait loop
-  while (!fs.existsSync(codeFile)) { updateStdout(); };
-  while (!fs.existsSync(stdoutFile)) { updateStdout(); };
+  // sleepFile is used as a dummy I/O op to mitigate unnecessary CPU usage
+  // (tried many I/O sync ops, writeFileSync() seems to be only one that is effective in reducing
+  // CPU usage, though apparently not so much on Windows)
+  while (!fs.existsSync(codeFile)) { updateStdout(); fs.writeFileSync(sleepFile, 'a'); };
+  while (!fs.existsSync(stdoutFile)) { updateStdout(); fs.writeFileSync(sleepFile, 'a'); };
 
   // At this point codeFile exists, but it's not necessarily flushed yet.
   // Keep reading it until it is.
@@ -1236,9 +1325,10 @@ function execSync(cmd, opts) {
 
   var stdout = fs.readFileSync(stdoutFile, 'utf8');
 
-  fs.unlinkSync(scriptFile);
-  fs.unlinkSync(stdoutFile);
-  fs.unlinkSync(codeFile);
+  _unlinkSync(scriptFile);
+  _unlinkSync(stdoutFile);
+  _unlinkSync(codeFile);
+  _unlinkSync(sleepFile);
 
   // True if successful, false if not
   var obj = {
@@ -1257,8 +1347,9 @@ function expand(list) {
   list.forEach(function(listEl) {
     // Wildcard present? 
     if (listEl.search(/\*/) > -1) {
-      for (file in _ls('', listEl))
+      _ls('', listEl).forEach(function(file) {
         expanded.push(file);
+      });
     } else {
       expanded.push(listEl);
     }
@@ -1290,3 +1381,33 @@ function extend(target) {
   
   return target;
 }
+
+// Normalizes _unlinkSync() across platforms to match Unix behavior, i.e.
+// file can be unlinked even if it's read-only, see joyent/node#3006
+function _unlinkSync(file) {
+  try {
+    fs.unlinkSync(file);
+  } catch(e) {
+    // Try to override file permission
+    if (e.code === 'EPERM') {
+      fs.chmodSync(file, '0666');
+      fs.unlinkSync(file);
+    } else {
+      throw e;
+    }
+  }
+}
+
+// Hack to determine if file has write permissions for current user
+// Avoids having to check user, group, etc, but it's probably slow
+function isWriteable(file) {
+  var writePermission = true;
+  try {
+    var __fd = fs.openSync(file, 'a');
+    fs.closeSync(__fd);
+  } catch(e) {
+    writePermission = false;
+  }
+
+  return writePermission;
+}
diff --git a/make.js b/make.js
index 33771aeb7..973cb445a 100755
--- a/make.js
+++ b/make.js
@@ -99,7 +99,7 @@ target.bundle = function() {
         'jpx.js',
         'bidi.js'];
 
-  if (!exists(BUILD_DIR))
+  if (!test('-d', BUILD_DIR))
     mkdir(BUILD_DIR);
 
   cd('src');
@@ -142,10 +142,10 @@ target.pagesrepo = function() {
   echo();
   echo('### Creating fresh clone of gh-pages');
 
-  if (!exists(BUILD_DIR))
+  if (!test('-d', BUILD_DIR))
     mkdir(BUILD_DIR);
 
-  if (!exists(GH_PAGES_DIR)) {
+  if (!test('-d', GH_PAGES_DIR)) {
     echo();
     echo('Cloning project repo...');
     echo('(This operation can take a while, depending on network conditions)');
@@ -277,10 +277,10 @@ target.firefox = function() {
   // We don't need pdf.js anymore since its inlined
   rm('-Rf', FIREFOX_BUILD_CONTENT_DIR + BUILD_DIR);
   // Remove '.DS_Store' and other hidden files
-  for (file in find(FIREFOX_BUILD_DIR)) {
+  find(FIREFOX_BUILD_DIR).forEach(function(file) {
     if (file.match(/^\./))
       rm('-f', file);
-  }
+  });
 
   // Update the build version number
   sed('-i', /PDFJSSCRIPT_VERSION/, EXTENSION_VERSION, FIREFOX_BUILD_DIR + '/install.rdf');
@@ -304,10 +304,10 @@ target.firefox = function() {
   // List all files for mozilla-central
   cd(FIREFOX_BUILD_DIR);
   var extensionFiles = '';
-  for (file in find(FIREFOX_MC_EXTENSION_FILES)) {
+  find(FIREFOX_MC_EXTENSION_FILES).forEach(function(file){
     if (test('-f', file))
       extensionFiles += file+'\n';
-  }
+  });
   extensionFiles.to('extension-files');
 };
 
@@ -357,7 +357,7 @@ target.chrome = function() {
 //
 target.test = function() {
   target.browsertest();
-  target.unittest();
+//  target.unittest();
 };
 
 //
@@ -371,14 +371,14 @@ target.browsertest = function() {
   var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
       PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
 
-  if (!exists('test/' + PDF_BROWSERS)) {
+  if (!test('-f', 'test/' + PDF_BROWSERS)) {
     echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
     echo('Try copying one of the examples in test/resources/browser_manifests/');
     exit(1);
   }
 
   cd('test');
-  exec(PYTHON_BIN + ' test.py --reftest --browserManifestFile=' + PDF_BROWSERS +
+  exec(PYTHON_BIN + ' -u test.py --reftest --browserManifestFile=' + PDF_BROWSERS +
     ' --manifestFile=' + PDF_TEST, {async: true});
 };
 
@@ -390,10 +390,37 @@ target.unittest = function() {
   echo();
   echo('### Running unit tests');
 
+  if (!which('make')) {
+    echo('make not found. Skipping unit tests...');
+    return;
+  }
+
   cd('test/unit');
   exec('make', {async: true});
 };
 
+//
+// make makeref
+//
+target.makeref = function() {
+  cd(ROOT_DIR);
+  echo();
+  echo('### Creating reference images');
+
+  var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
+      PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
+
+  if (!test('-f', 'test/' + PDF_BROWSERS)) {
+    echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
+    echo('Try copying one of the examples in test/resources/browser_manifests/');
+    exit(1);
+  }
+
+  cd('test');
+  exec(PYTHON_BIN + ' -u test.py --masterMode --noPrompts --browserManifestFile=' + PDF_BROWSERS,
+    {async: true});
+};
+
 
 ///////////////////////////////////////////////////////////////////////////////////////////
 //

From 0d216e62fed26096823fb421ef0eb3c940c4372b Mon Sep 17 00:00:00 2001
From: Artur Adib <arturadib@gmail.com>
Date: Tue, 27 Mar 2012 21:00:27 -0400
Subject: [PATCH 2/6] fix wnv_chinese link

---
 test/pdfs/wnv_chinese.pdf.link | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/pdfs/wnv_chinese.pdf.link b/test/pdfs/wnv_chinese.pdf.link
index fbbc81760..0bd2af8f6 100644
--- a/test/pdfs/wnv_chinese.pdf.link
+++ b/test/pdfs/wnv_chinese.pdf.link
@@ -1 +1 @@
-http://www.cdc.gov/ncidod/dvbid/westnile/languages/chinese.pdf
+http://web.archive.org/web/20110623114753/http://www.cdc.gov/ncidod/dvbid/westnile/languages/chinese.pdf

From 89ceead8b004754364e706eaa1316f8d4468b0c7 Mon Sep 17 00:00:00 2001
From: Artur Adib <arturadib@gmail.com>
Date: Wed, 28 Mar 2012 17:06:41 -0400
Subject: [PATCH 3/6] make bottest

---
 make.js | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/make.js b/make.js
index 973cb445a..13768ebb5 100755
--- a/make.js
+++ b/make.js
@@ -357,13 +357,22 @@ target.chrome = function() {
 //
 target.test = function() {
   target.browsertest();
-//  target.unittest();
+  target.unittest();
+};
+
+//
+// make bottest
+// (Special tests for the Github bot)
+//
+target.bottest = function() {
+  target.browsertest({noreftest: true});
+  // target.unittest();
 };
 
 //
 // make browsertest
 //
-target.browsertest = function() {
+target.browsertest = function(options) {
   cd(ROOT_DIR);
   echo();
   echo('### Running browser tests');
@@ -377,8 +386,10 @@ target.browsertest = function() {
     exit(1);
   }
 
+  var reftest = options.noreftest ? '' : '--reftest';
+
   cd('test');
-  exec(PYTHON_BIN + ' -u test.py --reftest --browserManifestFile=' + PDF_BROWSERS +
+  exec(PYTHON_BIN + ' -u test.py '+reftest+' --browserManifestFile=' + PDF_BROWSERS +
     ' --manifestFile=' + PDF_TEST, {async: true});
 };
 

From c0289497ec4c8112bd4d78aa18c4a6ba5b761e45 Mon Sep 17 00:00:00 2001
From: Artur Adib <arturadib@gmail.com>
Date: Wed, 28 Mar 2012 18:03:31 -0400
Subject: [PATCH 4/6] bumping ShellJS

---
 external/shelljs/README.md    |  9 ++++++--
 external/shelljs/make.js      |  2 +-
 external/shelljs/package.json | 35 +++++++++++++++++++++--------
 external/shelljs/shell.js     | 42 +++++++++++++++++++++--------------
 4 files changed, 59 insertions(+), 29 deletions(-)

diff --git a/external/shelljs/README.md b/external/shelljs/README.md
index 9f5386efe..af5f05127 100644
--- a/external/shelljs/README.md
+++ b/external/shelljs/README.md
@@ -138,7 +138,7 @@ Examples:
 ```javascript
 find('src', 'lib');
 find(['src', 'lib']); // same as above
-find('.').filter(function(file) { return file.match(/\.js$/); })
+find('.').filter(function(file) { return file.match(/\.js$/); });
 ```
 
 Returns array of all files (however deep) in the given paths.
@@ -325,6 +325,10 @@ When in synchronous mode returns the object `{ code:..., output:... }`, containi
 `output` (stdout + stderr)  and its exit `code`. Otherwise the `callback` gets the 
 arguments `(code, output)`.
 
+**Note:** For long-lived processes, it's best to run `exec()` asynchronously as
+the current synchronous implementation uses a lot of CPU. This should be getting
+fixed soon.
+
 ## Non-Unix commands
 
 
@@ -346,7 +350,8 @@ silent(true);
 silent(silentState); // restore old silent state
 ```
 
-Suppresses all output if `state = true`. Returns state if no arguments given.
+Suppresses all command output if `state = true`, except for `echo()` calls. 
+Returns state if no arguments given.
 
 ## Deprecated
 
diff --git a/external/shelljs/make.js b/external/shelljs/make.js
index 9c92dadc8..c495735bf 100644
--- a/external/shelljs/make.js
+++ b/external/shelljs/make.js
@@ -23,7 +23,7 @@ setTimeout(function() {
         if (oldTarget.done && !force)
           return;
         oldTarget.done = true;
-        return oldTarget(arguments);
+        return oldTarget.apply(oldTarget, arguments);
       }
 
     })(t, target[t]);
diff --git a/external/shelljs/package.json b/external/shelljs/package.json
index 99c0f5e64..4d2830cb9 100644
--- a/external/shelljs/package.json
+++ b/external/shelljs/package.json
@@ -1,12 +1,29 @@
-{ "name": "shelljs"
-, "version": "0.0.5pre2"
-, "author": "Artur Adib <aadib@mozilla.com>"
-, "description": "Portable Unix shell commands for Node.js"
-, "keywords": ["unix", "shell", "makefile", "make", "jake", "synchronous"]
-, "repository": "git://github.com/arturadib/shelljs"
-, "homepage": "http://github.com/arturadib/shelljs"
-, "main": "./shell.js"
-, "scripts": {
+{
+  "name": "shelljs",
+  "version": "0.0.5pre4",
+  "author": "Artur Adib <aadib@mozilla.com>",
+  "description": "Portable Unix shell commands for Node.js",
+  "keywords": [
+    "unix",
+    "shell",
+    "makefile",
+    "make",
+    "jake",
+    "synchronous"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/arturadib/shelljs.git"
+  },
+  "homepage": "http://github.com/arturadib/shelljs",
+  "main": "./shell.js",
+  "scripts": {
     "test": "node scripts/run-tests"
+  },
+  "dependencies": {},
+  "devDependencies": {},
+  "optionalDependencies": {},
+  "engines": {
+    "node": "*"
   }
 }
diff --git a/external/shelljs/shell.js b/external/shelljs/shell.js
index 9c1181fb6..92c49c54d 100644
--- a/external/shelljs/shell.js
+++ b/external/shelljs/shell.js
@@ -777,7 +777,7 @@ exports.which = wrap('which', _which);
 //@ like `.to()`.
 function _echo(options) {
   var messages = [].slice.call(arguments, 1);
-  log.apply(this, messages);
+  console.log.apply(this, messages);
   return ShellString(messages.join(' '));
 };
 exports.echo = wrap('echo', _echo);
@@ -809,6 +809,10 @@ exports.env = process.env;
 //@ When in synchronous mode returns the object `{ code:..., output:... }`, containing the program's 
 //@ `output` (stdout + stderr)  and its exit `code`. Otherwise the `callback` gets the 
 //@ arguments `(code, output)`.
+//@
+//@ **Note:** For long-lived processes, it's best to run `exec()` asynchronously as
+//@ the current synchronous implementation uses a lot of CPU. This should be getting
+//@ fixed soon.
 function _exec(command, options, callback) {
   if (!command)
     error('must specify command');
@@ -868,7 +872,8 @@ exports.error = function() {
 //@ silent(silentState); // restore old silent state
 //@ ```
 //@
-//@ Suppresses all output if `state = true`. Returns state if no arguments given.
+//@ Suppresses all command output if `state = true`, except for `echo()` calls. 
+//@ Returns state if no arguments given.
 exports.silent = function(_state) {
   if (typeof _state !== 'boolean')
     return state.silent;
@@ -1020,7 +1025,7 @@ function wrap(cmd, fn, options) {
     } catch (e) {
       if (!state.error) {
         // If state.error hasn't been set it's an error thrown by Node, not us - probably a bug...
-        console.log('maker.js: internal error');
+        console.log('shell.js: internal error');
         console.log(e.stack || e);
         process.exit(1);
       }
@@ -1028,7 +1033,7 @@ function wrap(cmd, fn, options) {
         throw e;
     }
 
-    state.currentCmd = 'maker.js';
+    state.currentCmd = 'shell.js';
     return retValue;
   }
 } // wrap
@@ -1151,8 +1156,7 @@ function rmdirSyncRecursive(dir, force) {
   try {
     result = fs.rmdirSync(dir);
   } catch(e) {
-    if (e.code === 'ENOTEMPTY')
-      error('directory not empty: ' + dir, true);
+    error('could not remove directory (code '+e.code+'): ' + dir, true);
   }
 
   return result;
@@ -1234,8 +1238,11 @@ function tempDir() {
 
 // Wrapper around exec() to enable echoing output to console in real time
 function execAsync(cmd, opts, callback) {
-  var output = '',
-      silent = 'silent' in opts ? opts.silent : state.silent;
+  var output = '';
+
+  var options = extend({
+    silent: state.silent
+  }, opts);
   
   var c = child.exec(cmd, {env: process.env}, function(err) {
     if (callback) 
@@ -1244,14 +1251,14 @@ function execAsync(cmd, opts, callback) {
 
   c.stdout.on('data', function(data) {
     output += data;
-    if (!silent)
-      write(data);
+    if (!options.silent)
+      process.stdout.write(data);
   });
 
   c.stderr.on('data', function(data) {
     output += data;
-    if (!silent)
-      write(data);
+    if (!options.silent)
+      process.stdout.write(data);
   });
 }
 
@@ -1325,11 +1332,12 @@ function execSync(cmd, opts) {
 
   var stdout = fs.readFileSync(stdoutFile, 'utf8');
 
-  _unlinkSync(scriptFile);
-  _unlinkSync(stdoutFile);
-  _unlinkSync(codeFile);
-  _unlinkSync(sleepFile);
-
+  // No biggie if we can't erase the files now -- they're in a temp dir anyway
+  try { _unlinkSync(scriptFile); } catch(e) {};
+  try { _unlinkSync(stdoutFile); } catch(e) {};
+  try { _unlinkSync(codeFile); } catch(e) {};
+  try { _unlinkSync(sleepFile); } catch(e) {};
+  
   // True if successful, false if not
   var obj = {
     code: code,

From 3740bed7d79374b2284c1d43836db334d450e603 Mon Sep 17 00:00:00 2001
From: Artur Adib <arturadib@gmail.com>
Date: Thu, 29 Mar 2012 15:04:12 -0400
Subject: [PATCH 5/6] minor fixes

---
 make.js | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/make.js b/make.js
index 13768ebb5..05ed2b39d 100755
--- a/make.js
+++ b/make.js
@@ -386,7 +386,7 @@ target.browsertest = function(options) {
     exit(1);
   }
 
-  var reftest = options.noreftest ? '' : '--reftest';
+  var reftest = (options && options.noreftest) ? '' : '--reftest';
 
   cd('test');
   exec(PYTHON_BIN + ' -u test.py '+reftest+' --browserManifestFile=' + PDF_BROWSERS +
@@ -427,6 +427,28 @@ target.makeref = function() {
     exit(1);
   }
 
+  cd('test');
+  exec(PYTHON_BIN + ' -u test.py --masterMode --browserManifestFile=' + PDF_BROWSERS,
+    {async: true});
+};
+
+//
+// make botmakeref
+//
+target.botmakeref = function() {
+  cd(ROOT_DIR);
+  echo();
+  echo('### Creating reference images');
+
+  var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
+      PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
+
+  if (!test('-f', 'test/' + PDF_BROWSERS)) {
+    echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
+    echo('Try copying one of the examples in test/resources/browser_manifests/');
+    exit(1);
+  }
+
   cd('test');
   exec(PYTHON_BIN + ' -u test.py --masterMode --noPrompts --browserManifestFile=' + PDF_BROWSERS,
     {async: true});

From fbc0ed45de479000464ee53ee775dd7dc5447e1d Mon Sep 17 00:00:00 2001
From: Artur Adib <arturadib@gmail.com>
Date: Thu, 29 Mar 2012 15:12:27 -0400
Subject: [PATCH 6/6] oops: remove makeref

---
 make.js | 22 ----------------------
 1 file changed, 22 deletions(-)

diff --git a/make.js b/make.js
index 05ed2b39d..7f48e6d93 100755
--- a/make.js
+++ b/make.js
@@ -410,28 +410,6 @@ target.unittest = function() {
   exec('make', {async: true});
 };
 
-//
-// make makeref
-//
-target.makeref = function() {
-  cd(ROOT_DIR);
-  echo();
-  echo('### Creating reference images');
-
-  var PDF_TEST = env['PDF_TEST'] || 'test_manifest.json',
-      PDF_BROWSERS = env['PDF_BROWSERS'] || 'resources/browser_manifests/browser_manifest.json';
-
-  if (!test('-f', 'test/' + PDF_BROWSERS)) {
-    echo('Browser manifest file test/' + PDF_BROWSERS + ' does not exist.');
-    echo('Try copying one of the examples in test/resources/browser_manifests/');
-    exit(1);
-  }
-
-  cd('test');
-  exec(PYTHON_BIN + ' -u test.py --masterMode --browserManifestFile=' + PDF_BROWSERS,
-    {async: true});
-};
-
 //
 // make botmakeref
 //