worker.js 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. 'use strict';
  2. var _fs = require('fs');
  3. var _fs2 = _interopRequireDefault(_fs);
  4. var _module = require('module');
  5. var _module2 = _interopRequireDefault(_module);
  6. var _loaderRunner = require('loader-runner');
  7. var _loaderRunner2 = _interopRequireDefault(_loaderRunner);
  8. var _queue = require('neo-async/queue');
  9. var _queue2 = _interopRequireDefault(_queue);
  10. var _readBuffer = require('./readBuffer');
  11. var _readBuffer2 = _interopRequireDefault(_readBuffer);
  12. function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
  13. const writePipe = _fs2.default.createWriteStream(null, { fd: 3 }); /* global require */
  14. /* eslint-disable no-console */
  15. const readPipe = _fs2.default.createReadStream(null, { fd: 4 });
  16. writePipe.on('finish', onTerminateWrite);
  17. readPipe.on('end', onTerminateRead);
  18. writePipe.on('close', onTerminateWrite);
  19. readPipe.on('close', onTerminateRead);
  20. readPipe.on('error', onError);
  21. writePipe.on('error', onError);
  22. const PARALLEL_JOBS = +process.argv[2] || 20;
  23. let terminated = false;
  24. let nextQuestionId = 0;
  25. const callbackMap = Object.create(null);
  26. function onError(error) {
  27. console.error(error);
  28. }
  29. function onTerminateRead() {
  30. terminateRead();
  31. }
  32. function onTerminateWrite() {
  33. terminateWrite();
  34. }
  35. function writePipeWrite(...args) {
  36. if (!terminated) {
  37. writePipe.write(...args);
  38. }
  39. }
  40. function writePipeCork() {
  41. if (!terminated) {
  42. writePipe.cork();
  43. }
  44. }
  45. function writePipeUncork() {
  46. if (!terminated) {
  47. writePipe.uncork();
  48. }
  49. }
  50. function terminateRead() {
  51. terminated = true;
  52. readPipe.removeAllListeners();
  53. }
  54. function terminateWrite() {
  55. terminated = true;
  56. writePipe.removeAllListeners();
  57. }
  58. function terminate() {
  59. terminateRead();
  60. terminateWrite();
  61. }
  62. function toErrorObj(err) {
  63. return {
  64. message: err.message,
  65. details: err.details,
  66. stack: err.stack,
  67. hideStack: err.hideStack
  68. };
  69. }
  70. function toNativeError(obj) {
  71. if (!obj) return null;
  72. const err = new Error(obj.message);
  73. err.details = obj.details;
  74. err.missing = obj.missing;
  75. return err;
  76. }
  77. function writeJson(data) {
  78. writePipeCork();
  79. process.nextTick(() => {
  80. writePipeUncork();
  81. });
  82. const lengthBuffer = Buffer.alloc(4);
  83. const messageBuffer = Buffer.from(JSON.stringify(data), 'utf-8');
  84. lengthBuffer.writeInt32BE(messageBuffer.length, 0);
  85. writePipeWrite(lengthBuffer);
  86. writePipeWrite(messageBuffer);
  87. }
  88. const queue = (0, _queue2.default)(({ id, data }, taskCallback) => {
  89. try {
  90. _loaderRunner2.default.runLoaders({
  91. loaders: data.loaders,
  92. resource: data.resource,
  93. readResource: _fs2.default.readFile.bind(_fs2.default),
  94. context: {
  95. version: 2,
  96. resolve: (context, request, callback) => {
  97. callbackMap[nextQuestionId] = callback;
  98. writeJson({
  99. type: 'resolve',
  100. id,
  101. questionId: nextQuestionId,
  102. context,
  103. request
  104. });
  105. nextQuestionId += 1;
  106. },
  107. emitWarning: warning => {
  108. writeJson({
  109. type: 'emitWarning',
  110. id,
  111. data: toErrorObj(warning)
  112. });
  113. },
  114. emitError: error => {
  115. writeJson({
  116. type: 'emitError',
  117. id,
  118. data: toErrorObj(error)
  119. });
  120. },
  121. exec: (code, filename) => {
  122. const module = new _module2.default(filename, undefined);
  123. module.paths = _module2.default._nodeModulePaths(undefined.context); // eslint-disable-line no-underscore-dangle
  124. module.filename = filename;
  125. module._compile(code, filename); // eslint-disable-line no-underscore-dangle
  126. return module.exports;
  127. },
  128. options: {
  129. context: data.optionsContext
  130. },
  131. webpack: true,
  132. 'thread-loader': true,
  133. sourceMap: data.sourceMap,
  134. target: data.target,
  135. minimize: data.minimize,
  136. resourceQuery: data.resourceQuery
  137. }
  138. }, (err, lrResult) => {
  139. const {
  140. result,
  141. cacheable,
  142. fileDependencies,
  143. contextDependencies
  144. } = lrResult;
  145. const buffersToSend = [];
  146. const convertedResult = Array.isArray(result) && result.map(item => {
  147. const isBuffer = Buffer.isBuffer(item);
  148. if (isBuffer) {
  149. buffersToSend.push(item);
  150. return {
  151. buffer: true
  152. };
  153. }
  154. if (typeof item === 'string') {
  155. const stringBuffer = Buffer.from(item, 'utf-8');
  156. buffersToSend.push(stringBuffer);
  157. return {
  158. buffer: true,
  159. string: true
  160. };
  161. }
  162. return {
  163. data: item
  164. };
  165. });
  166. writeJson({
  167. type: 'job',
  168. id,
  169. error: err && toErrorObj(err),
  170. result: {
  171. result: convertedResult,
  172. cacheable,
  173. fileDependencies,
  174. contextDependencies
  175. },
  176. data: buffersToSend.map(buffer => buffer.length)
  177. });
  178. buffersToSend.forEach(buffer => {
  179. writePipeWrite(buffer);
  180. });
  181. setImmediate(taskCallback);
  182. });
  183. } catch (e) {
  184. writeJson({
  185. type: 'job',
  186. id,
  187. error: toErrorObj(e)
  188. });
  189. taskCallback();
  190. }
  191. }, PARALLEL_JOBS);
  192. function dispose() {
  193. terminate();
  194. queue.kill();
  195. process.exit(0);
  196. }
  197. function onMessage(message) {
  198. try {
  199. const { type, id } = message;
  200. switch (type) {
  201. case 'job':
  202. {
  203. queue.push(message);
  204. break;
  205. }
  206. case 'result':
  207. {
  208. const { error, result } = message;
  209. const callback = callbackMap[id];
  210. if (callback) {
  211. const nativeError = toNativeError(error);
  212. callback(nativeError, result);
  213. } else {
  214. console.error(`Worker got unexpected result id ${id}`);
  215. }
  216. delete callbackMap[id];
  217. break;
  218. }
  219. case 'warmup':
  220. {
  221. const { requires } = message;
  222. // load modules into process
  223. requires.forEach(r => require(r)); // eslint-disable-line import/no-dynamic-require, global-require
  224. break;
  225. }
  226. default:
  227. {
  228. console.error(`Worker got unexpected job type ${type}`);
  229. break;
  230. }
  231. }
  232. } catch (e) {
  233. console.error(`Error in worker ${e}`);
  234. }
  235. }
  236. function readNextMessage() {
  237. (0, _readBuffer2.default)(readPipe, 4, (lengthReadError, lengthBuffer) => {
  238. if (lengthReadError) {
  239. console.error(`Failed to communicate with main process (read length) ${lengthReadError}`);
  240. return;
  241. }
  242. const length = lengthBuffer.length && lengthBuffer.readInt32BE(0);
  243. if (length === 0) {
  244. // worker should dispose and exit
  245. dispose();
  246. return;
  247. }
  248. (0, _readBuffer2.default)(readPipe, length, (messageError, messageBuffer) => {
  249. if (terminated) {
  250. return;
  251. }
  252. if (messageError) {
  253. console.error(`Failed to communicate with main process (read message) ${messageError}`);
  254. return;
  255. }
  256. const messageString = messageBuffer.toString('utf-8');
  257. const message = JSON.parse(messageString);
  258. onMessage(message);
  259. setImmediate(() => readNextMessage());
  260. });
  261. });
  262. }
  263. // start reading messages from main process
  264. readNextMessage();