dispatcher.js 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515
  1. "use strict";
  2. var __defProp = Object.defineProperty;
  3. var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
  4. var __getOwnPropNames = Object.getOwnPropertyNames;
  5. var __hasOwnProp = Object.prototype.hasOwnProperty;
  6. var __export = (target, all) => {
  7. for (var name in all)
  8. __defProp(target, name, { get: all[name], enumerable: true });
  9. };
  10. var __copyProps = (to, from, except, desc) => {
  11. if (from && typeof from === "object" || typeof from === "function") {
  12. for (let key of __getOwnPropNames(from))
  13. if (!__hasOwnProp.call(to, key) && key !== except)
  14. __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
  15. }
  16. return to;
  17. };
  18. var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
  19. var dispatcher_exports = {};
  20. __export(dispatcher_exports, {
  21. Dispatcher: () => Dispatcher
  22. });
  23. module.exports = __toCommonJS(dispatcher_exports);
  24. var import_utils = require("playwright-core/lib/utils");
  25. var import_utils2 = require("playwright-core/lib/utils");
  26. var import_rebase = require("./rebase");
  27. var import_workerHost = require("./workerHost");
  28. var import_ipc = require("../common/ipc");
  29. var import_internalReporter = require("../reporters/internalReporter");
  30. var import_util = require("../util");
  31. class Dispatcher {
  32. constructor(config, reporter, failureTracker) {
  33. this._workerSlots = [];
  34. this._queue = [];
  35. this._workerLimitPerProjectId = /* @__PURE__ */ new Map();
  36. this._queuedOrRunningHashCount = /* @__PURE__ */ new Map();
  37. this._finished = new import_utils.ManualPromise();
  38. this._isStopped = true;
  39. this._extraEnvByProjectId = /* @__PURE__ */ new Map();
  40. this._producedEnvByProjectId = /* @__PURE__ */ new Map();
  41. this._config = config;
  42. this._reporter = reporter;
  43. this._failureTracker = failureTracker;
  44. for (const project of config.projects) {
  45. if (project.workers)
  46. this._workerLimitPerProjectId.set(project.id, project.workers);
  47. }
  48. }
  49. _findFirstJobToRun() {
  50. for (let index = 0; index < this._queue.length; index++) {
  51. const job = this._queue[index];
  52. const projectIdWorkerLimit = this._workerLimitPerProjectId.get(job.projectId);
  53. if (!projectIdWorkerLimit)
  54. return index;
  55. const runningWorkersWithSameProjectId = this._workerSlots.filter((w) => w.busy && w.worker && w.worker.projectId() === job.projectId).length;
  56. if (runningWorkersWithSameProjectId < projectIdWorkerLimit)
  57. return index;
  58. }
  59. return -1;
  60. }
  61. _scheduleJob() {
  62. if (this._isStopped)
  63. return;
  64. const jobIndex = this._findFirstJobToRun();
  65. if (jobIndex === -1)
  66. return;
  67. const job = this._queue[jobIndex];
  68. let workerIndex = this._workerSlots.findIndex((w) => !w.busy && w.worker && w.worker.hash() === job.workerHash && !w.worker.didSendStop());
  69. if (workerIndex === -1)
  70. workerIndex = this._workerSlots.findIndex((w) => !w.busy);
  71. if (workerIndex === -1) {
  72. return;
  73. }
  74. this._queue.splice(jobIndex, 1);
  75. const jobDispatcher = new JobDispatcher(job, this._config, this._reporter, this._failureTracker, () => this.stop().catch(() => {
  76. }));
  77. this._workerSlots[workerIndex].busy = true;
  78. this._workerSlots[workerIndex].jobDispatcher = jobDispatcher;
  79. void this._runJobInWorker(workerIndex, jobDispatcher).then(() => {
  80. this._workerSlots[workerIndex].jobDispatcher = void 0;
  81. this._workerSlots[workerIndex].busy = false;
  82. this._checkFinished();
  83. this._scheduleJob();
  84. });
  85. }
  86. async _runJobInWorker(index, jobDispatcher) {
  87. const job = jobDispatcher.job;
  88. if (jobDispatcher.skipWholeJob())
  89. return;
  90. let worker = this._workerSlots[index].worker;
  91. if (worker && (worker.hash() !== job.workerHash || worker.didSendStop())) {
  92. await worker.stop();
  93. worker = void 0;
  94. if (this._isStopped)
  95. return;
  96. }
  97. let startError;
  98. if (!worker) {
  99. worker = this._createWorker(job, index, (0, import_ipc.serializeConfig)(this._config, true));
  100. this._workerSlots[index].worker = worker;
  101. worker.on("exit", () => this._workerSlots[index].worker = void 0);
  102. startError = await worker.start();
  103. if (this._isStopped)
  104. return;
  105. }
  106. if (startError)
  107. jobDispatcher.onExit(startError);
  108. else
  109. jobDispatcher.runInWorker(worker);
  110. const result = await jobDispatcher.jobResult;
  111. this._updateCounterForWorkerHash(job.workerHash, -1);
  112. if (result.didFail)
  113. void worker.stop(
  114. true
  115. /* didFail */
  116. );
  117. else if (this._isWorkerRedundant(worker))
  118. void worker.stop();
  119. if (!this._isStopped && result.newJob) {
  120. this._queue.unshift(result.newJob);
  121. this._updateCounterForWorkerHash(result.newJob.workerHash, 1);
  122. }
  123. }
  124. _checkFinished() {
  125. if (this._finished.isDone())
  126. return;
  127. if (this._queue.length && !this._isStopped)
  128. return;
  129. if (this._workerSlots.some((w) => w.busy))
  130. return;
  131. this._finished.resolve();
  132. }
  133. _isWorkerRedundant(worker) {
  134. let workersWithSameHash = 0;
  135. for (const slot of this._workerSlots) {
  136. if (slot.worker && !slot.worker.didSendStop() && slot.worker.hash() === worker.hash())
  137. workersWithSameHash++;
  138. }
  139. return workersWithSameHash > this._queuedOrRunningHashCount.get(worker.hash());
  140. }
  141. _updateCounterForWorkerHash(hash, delta) {
  142. this._queuedOrRunningHashCount.set(hash, delta + (this._queuedOrRunningHashCount.get(hash) || 0));
  143. }
  144. async run(testGroups, extraEnvByProjectId) {
  145. this._extraEnvByProjectId = extraEnvByProjectId;
  146. this._queue = testGroups;
  147. for (const group of testGroups)
  148. this._updateCounterForWorkerHash(group.workerHash, 1);
  149. this._isStopped = false;
  150. this._workerSlots = [];
  151. if (this._failureTracker.hasReachedMaxFailures())
  152. void this.stop();
  153. for (let i = 0; i < this._config.config.workers; i++)
  154. this._workerSlots.push({ busy: false });
  155. for (let i = 0; i < this._workerSlots.length; i++)
  156. this._scheduleJob();
  157. this._checkFinished();
  158. await this._finished;
  159. }
  160. _createWorker(testGroup, parallelIndex, loaderData) {
  161. const projectConfig = this._config.projects.find((p) => p.id === testGroup.projectId);
  162. const outputDir = projectConfig.project.outputDir;
  163. const worker = new import_workerHost.WorkerHost(testGroup, {
  164. parallelIndex,
  165. config: loaderData,
  166. extraEnv: this._extraEnvByProjectId.get(testGroup.projectId) || {},
  167. outputDir,
  168. pauseOnError: this._failureTracker.pauseOnError(),
  169. pauseAtEnd: this._failureTracker.pauseAtEnd(projectConfig)
  170. });
  171. const handleOutput = (params) => {
  172. const chunk = chunkFromParams(params);
  173. if (worker.didFail()) {
  174. return { chunk };
  175. }
  176. const currentlyRunning = this._workerSlots[parallelIndex].jobDispatcher?.currentlyRunning();
  177. if (!currentlyRunning)
  178. return { chunk };
  179. return { chunk, test: currentlyRunning.test, result: currentlyRunning.result };
  180. };
  181. worker.on("stdOut", (params) => {
  182. const { chunk, test, result } = handleOutput(params);
  183. result?.stdout.push(chunk);
  184. this._reporter.onStdOut?.(chunk, test, result);
  185. });
  186. worker.on("stdErr", (params) => {
  187. const { chunk, test, result } = handleOutput(params);
  188. result?.stderr.push(chunk);
  189. this._reporter.onStdErr?.(chunk, test, result);
  190. });
  191. worker.on("teardownErrors", (params) => {
  192. this._failureTracker.onWorkerError();
  193. for (const error of params.fatalErrors)
  194. this._reporter.onError?.(error);
  195. });
  196. worker.on("exit", () => {
  197. const producedEnv = this._producedEnvByProjectId.get(testGroup.projectId) || {};
  198. this._producedEnvByProjectId.set(testGroup.projectId, { ...producedEnv, ...worker.producedEnv() });
  199. });
  200. return worker;
  201. }
  202. producedEnvByProjectId() {
  203. return this._producedEnvByProjectId;
  204. }
  205. async stop() {
  206. if (this._isStopped)
  207. return;
  208. this._isStopped = true;
  209. await Promise.all(this._workerSlots.map(({ worker }) => worker?.stop()));
  210. this._checkFinished();
  211. }
  212. }
  213. class JobDispatcher {
  214. constructor(job, config, reporter, failureTracker, stopCallback) {
  215. this.jobResult = new import_utils.ManualPromise();
  216. this._listeners = [];
  217. this._failedTests = /* @__PURE__ */ new Set();
  218. this._failedWithNonRetriableError = /* @__PURE__ */ new Set();
  219. this._remainingByTestId = /* @__PURE__ */ new Map();
  220. this._dataByTestId = /* @__PURE__ */ new Map();
  221. this._parallelIndex = 0;
  222. this._workerIndex = 0;
  223. this.job = job;
  224. this._config = config;
  225. this._reporter = reporter;
  226. this._failureTracker = failureTracker;
  227. this._stopCallback = stopCallback;
  228. this._remainingByTestId = new Map(this.job.tests.map((e) => [e.id, e]));
  229. }
  230. _onTestBegin(params) {
  231. const test = this._remainingByTestId.get(params.testId);
  232. if (!test) {
  233. return;
  234. }
  235. const result = test._appendTestResult();
  236. this._dataByTestId.set(test.id, { test, result, steps: /* @__PURE__ */ new Map() });
  237. result.parallelIndex = this._parallelIndex;
  238. result.workerIndex = this._workerIndex;
  239. result.startTime = new Date(params.startWallTime);
  240. this._reporter.onTestBegin?.(test, result);
  241. this._currentlyRunning = { test, result };
  242. }
  243. _onTestEnd(params) {
  244. if (this._failureTracker.hasReachedMaxFailures()) {
  245. params.status = "interrupted";
  246. params.errors = [];
  247. }
  248. const data = this._dataByTestId.get(params.testId);
  249. if (!data) {
  250. return;
  251. }
  252. this._dataByTestId.delete(params.testId);
  253. this._remainingByTestId.delete(params.testId);
  254. const { result, test } = data;
  255. result.duration = params.duration;
  256. result.errors = params.errors;
  257. result.error = result.errors[0];
  258. result.status = params.status;
  259. result.annotations = params.annotations;
  260. test.annotations = [...params.annotations];
  261. test.expectedStatus = params.expectedStatus;
  262. test.timeout = params.timeout;
  263. const isFailure = result.status !== "skipped" && result.status !== test.expectedStatus;
  264. if (isFailure)
  265. this._failedTests.add(test);
  266. if (params.hasNonRetriableError)
  267. this._addNonretriableTestAndSerialModeParents(test);
  268. this._reportTestEnd(test, result);
  269. this._currentlyRunning = void 0;
  270. }
  271. _addNonretriableTestAndSerialModeParents(test) {
  272. this._failedWithNonRetriableError.add(test);
  273. for (let parent = test.parent; parent; parent = parent.parent) {
  274. if (parent._parallelMode === "serial")
  275. this._failedWithNonRetriableError.add(parent);
  276. }
  277. }
  278. _onStepBegin(params) {
  279. const data = this._dataByTestId.get(params.testId);
  280. if (!data) {
  281. return;
  282. }
  283. const { result, steps, test } = data;
  284. const parentStep = params.parentStepId ? steps.get(params.parentStepId) : void 0;
  285. const step = {
  286. title: params.title,
  287. titlePath: () => {
  288. const parentPath = parentStep?.titlePath() || [];
  289. return [...parentPath, params.title];
  290. },
  291. parent: parentStep,
  292. category: params.category,
  293. startTime: new Date(params.wallTime),
  294. duration: -1,
  295. steps: [],
  296. attachments: [],
  297. annotations: [],
  298. location: params.location
  299. };
  300. steps.set(params.stepId, step);
  301. (parentStep || result).steps.push(step);
  302. this._reporter.onStepBegin?.(test, result, step);
  303. }
  304. _onStepEnd(params) {
  305. const data = this._dataByTestId.get(params.testId);
  306. if (!data) {
  307. return;
  308. }
  309. const { result, steps, test } = data;
  310. const step = steps.get(params.stepId);
  311. if (!step) {
  312. this._reporter.onStdErr?.("Internal error: step end without step begin: " + params.stepId, test, result);
  313. return;
  314. }
  315. step.duration = params.wallTime - step.startTime.getTime();
  316. if (params.error)
  317. step.error = params.error;
  318. if (params.suggestedRebaseline)
  319. (0, import_rebase.addSuggestedRebaseline)(step.location, params.suggestedRebaseline);
  320. step.annotations = params.annotations;
  321. steps.delete(params.stepId);
  322. this._reporter.onStepEnd?.(test, result, step);
  323. }
  324. _onAttach(params) {
  325. const data = this._dataByTestId.get(params.testId);
  326. if (!data) {
  327. return;
  328. }
  329. const attachment = {
  330. name: params.name,
  331. path: params.path,
  332. contentType: params.contentType,
  333. body: params.body !== void 0 ? Buffer.from(params.body, "base64") : void 0
  334. };
  335. data.result.attachments.push(attachment);
  336. if (params.stepId) {
  337. const step = data.steps.get(params.stepId);
  338. if (step)
  339. step.attachments.push(attachment);
  340. else
  341. this._reporter.onStdErr?.("Internal error: step id not found: " + params.stepId);
  342. }
  343. }
  344. _failTestWithErrors(test, errors) {
  345. const runData = this._dataByTestId.get(test.id);
  346. let result;
  347. if (runData) {
  348. result = runData.result;
  349. } else {
  350. result = test._appendTestResult();
  351. this._reporter.onTestBegin?.(test, result);
  352. }
  353. result.errors = [...errors];
  354. result.error = result.errors[0];
  355. result.status = errors.length ? "failed" : "skipped";
  356. this._reportTestEnd(test, result);
  357. this._failedTests.add(test);
  358. }
  359. _massSkipTestsFromRemaining(testIds, errors) {
  360. for (const test of this._remainingByTestId.values()) {
  361. if (!testIds.has(test.id))
  362. continue;
  363. if (!this._failureTracker.hasReachedMaxFailures()) {
  364. this._failTestWithErrors(test, errors);
  365. errors = [];
  366. }
  367. this._remainingByTestId.delete(test.id);
  368. }
  369. if (errors.length) {
  370. this._failureTracker.onWorkerError();
  371. for (const error of errors)
  372. this._reporter.onError?.(error);
  373. }
  374. }
  375. _onDone(params) {
  376. if (!this._remainingByTestId.size && !this._failedTests.size && !params.fatalErrors.length && !params.skipTestsDueToSetupFailure.length && !params.fatalUnknownTestIds && !params.unexpectedExitError && !params.stoppedDueToUnhandledErrorInTestFail) {
  377. this._finished({ didFail: false });
  378. return;
  379. }
  380. for (const testId of params.fatalUnknownTestIds || []) {
  381. const test = this._remainingByTestId.get(testId);
  382. if (test) {
  383. this._remainingByTestId.delete(testId);
  384. this._failTestWithErrors(test, [{ message: `Test not found in the worker process. Make sure test title does not change.` }]);
  385. }
  386. }
  387. if (params.fatalErrors.length) {
  388. this._massSkipTestsFromRemaining(new Set(this._remainingByTestId.keys()), params.fatalErrors);
  389. }
  390. this._massSkipTestsFromRemaining(new Set(params.skipTestsDueToSetupFailure), []);
  391. if (params.unexpectedExitError) {
  392. if (this._currentlyRunning)
  393. this._massSkipTestsFromRemaining(/* @__PURE__ */ new Set([this._currentlyRunning.test.id]), [params.unexpectedExitError]);
  394. else
  395. this._massSkipTestsFromRemaining(new Set(this._remainingByTestId.keys()), [params.unexpectedExitError]);
  396. }
  397. const retryCandidates = /* @__PURE__ */ new Set();
  398. const serialSuitesWithFailures = /* @__PURE__ */ new Set();
  399. for (const failedTest of this._failedTests) {
  400. if (this._failedWithNonRetriableError.has(failedTest))
  401. continue;
  402. retryCandidates.add(failedTest);
  403. let outermostSerialSuite;
  404. for (let parent = failedTest.parent; parent; parent = parent.parent) {
  405. if (parent._parallelMode === "serial")
  406. outermostSerialSuite = parent;
  407. }
  408. if (outermostSerialSuite && !this._failedWithNonRetriableError.has(outermostSerialSuite))
  409. serialSuitesWithFailures.add(outermostSerialSuite);
  410. }
  411. const testsBelongingToSomeSerialSuiteWithFailures = [...this._remainingByTestId.values()].filter((test) => {
  412. let parent = test.parent;
  413. while (parent && !serialSuitesWithFailures.has(parent))
  414. parent = parent.parent;
  415. return !!parent;
  416. });
  417. this._massSkipTestsFromRemaining(new Set(testsBelongingToSomeSerialSuiteWithFailures.map((test) => test.id)), []);
  418. for (const serialSuite of serialSuitesWithFailures) {
  419. serialSuite.allTests().forEach((test) => retryCandidates.add(test));
  420. }
  421. const remaining = [...this._remainingByTestId.values()];
  422. for (const test of retryCandidates) {
  423. if (test.results.length < test.retries + 1)
  424. remaining.push(test);
  425. }
  426. const newJob = remaining.length ? { ...this.job, tests: remaining } : void 0;
  427. this._finished({ didFail: true, newJob });
  428. }
  429. onExit(data) {
  430. const unexpectedExitError = data.unexpectedly ? {
  431. message: `Error: worker process exited unexpectedly (code=${data.code}, signal=${data.signal})`
  432. } : void 0;
  433. this._onDone({ skipTestsDueToSetupFailure: [], fatalErrors: [], unexpectedExitError });
  434. }
  435. _finished(result) {
  436. import_utils.eventsHelper.removeEventListeners(this._listeners);
  437. this.jobResult.resolve(result);
  438. }
  439. runInWorker(worker) {
  440. this._parallelIndex = worker.parallelIndex;
  441. this._workerIndex = worker.workerIndex;
  442. const runPayload = {
  443. file: this.job.requireFile,
  444. entries: this.job.tests.map((test) => {
  445. return { testId: test.id, retry: test.results.length };
  446. })
  447. };
  448. worker.runTestGroup(runPayload);
  449. this._listeners = [
  450. import_utils.eventsHelper.addEventListener(worker, "testBegin", this._onTestBegin.bind(this)),
  451. import_utils.eventsHelper.addEventListener(worker, "testEnd", this._onTestEnd.bind(this)),
  452. import_utils.eventsHelper.addEventListener(worker, "stepBegin", this._onStepBegin.bind(this)),
  453. import_utils.eventsHelper.addEventListener(worker, "stepEnd", this._onStepEnd.bind(this)),
  454. import_utils.eventsHelper.addEventListener(worker, "attach", this._onAttach.bind(this)),
  455. import_utils.eventsHelper.addEventListener(worker, "testPaused", this._onTestPaused.bind(this, worker)),
  456. import_utils.eventsHelper.addEventListener(worker, "done", this._onDone.bind(this)),
  457. import_utils.eventsHelper.addEventListener(worker, "exit", this.onExit.bind(this))
  458. ];
  459. }
  460. _onTestPaused(worker, params) {
  461. const sendMessage = async (message) => {
  462. try {
  463. if (this.jobResult.isDone())
  464. throw new Error("Test has already stopped");
  465. const response = await worker.sendCustomMessage({ testId: params.testId, request: message.request });
  466. if (response.error)
  467. (0, import_internalReporter.addLocationAndSnippetToError)(this._config.config, response.error);
  468. return response;
  469. } catch (e) {
  470. const error = (0, import_util.serializeError)(e);
  471. (0, import_internalReporter.addLocationAndSnippetToError)(this._config.config, error);
  472. return { response: void 0, error };
  473. }
  474. };
  475. for (const error of params.errors)
  476. (0, import_internalReporter.addLocationAndSnippetToError)(this._config.config, error);
  477. this._failureTracker.onTestPaused?.({ ...params, sendMessage });
  478. }
  479. skipWholeJob() {
  480. const allTestsSkipped = this.job.tests.every((test) => test.expectedStatus === "skipped");
  481. if (allTestsSkipped && !this._failureTracker.hasReachedMaxFailures()) {
  482. for (const test of this.job.tests) {
  483. const result = test._appendTestResult();
  484. this._reporter.onTestBegin?.(test, result);
  485. result.status = "skipped";
  486. result.annotations = [...test.annotations];
  487. this._reportTestEnd(test, result);
  488. }
  489. return true;
  490. }
  491. return false;
  492. }
  493. currentlyRunning() {
  494. return this._currentlyRunning;
  495. }
  496. _reportTestEnd(test, result) {
  497. this._reporter.onTestEnd?.(test, result);
  498. const hadMaxFailures = this._failureTracker.hasReachedMaxFailures();
  499. this._failureTracker.onTestEnd(test, result);
  500. if (this._failureTracker.hasReachedMaxFailures()) {
  501. this._stopCallback();
  502. if (!hadMaxFailures)
  503. this._reporter.onError?.({ message: import_utils2.colors.red(`Testing stopped early after ${this._failureTracker.maxFailures()} maximum allowed failures.`) });
  504. }
  505. }
  506. }
  507. function chunkFromParams(params) {
  508. if (typeof params.text === "string")
  509. return params.text;
  510. return Buffer.from(params.buffer, "base64");
  511. }
  512. // Annotate the CommonJS export names for ESM import in node:
  513. 0 && (module.exports = {
  514. Dispatcher
  515. });