Tools/ChangeLog

22
33 Reviewed by NOBODY (OOPS!).
44
 5 nrwt: allow for multiple http shards
 6 https://bugs.webkit.org/show_bug.cgi?id=63116
 7
 8 This modifies the sharding logic to support multiple http
 9 shards, but for now we clamp to one shard until we can test
 10 perf impact and flakiness impact.
 11
 12 * Scripts/webkitpy/layout_tests/layout_package/manager.py:
 13 * Scripts/webkitpy/layout_tests/layout_package/manager_unittest.py:
 14
 152011-06-21 Dirk Pranke <dpranke@chromium.org>
 16
 17 Reviewed by NOBODY (OOPS!).
 18
 19 nrwt: make sharding tests needing locks less hard-coded
 20 https://bugs.webkit.org/show_bug.cgi?id=63112
 21
 22 This change also changes the manager logic so that it will
 23 drop the server lock as soon as all of the shards requiring
 24 the lock have completed.
 25
 26 * Scripts/webkitpy/layout_tests/layout_package/manager.py:
 27 * Scripts/webkitpy/layout_tests/layout_package/manager_unittest.py:
 28
 292011-06-21 Dirk Pranke <dpranke@chromium.org>
 30
 31 Reviewed by NOBODY (OOPS!).
 32
533 nrwt: make sharding tests needing locks less hard-coded
634 https://bugs.webkit.org/show_bug.cgi?id=63112
735

Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py

@@class Manager:
533533 return self._expectations.has_modifier(test_file,
534534 test_expectations.SLOW)
535535
536  def _shard_tests(self, test_files, use_real_shards):
 536 def _shard_tests(self, test_files, num_workers, fully_parallel):
537537 """Groups tests into batches.
538538 This helps ensure that tests that depend on each other (aka bad tests!)
539539 continue to run together as most cross-tests dependencies tend to
540  occur within the same directory. If use_real_shards is False, we
541  put each (non-HTTP/websocket) test into its own shard for maximum
542  concurrency instead of trying to do any sort of real sharding.
543 
 540 occur within the same directory.
544541 Return:
545542 Two lists of lists of TestInput objects. The first list should
546543 only be run under the server lock, the second can be run whenever.
547544 """
548  # FIXME: We still need to support multiple locked shards.
 545
 546 # FIXME: Move all of the sharding logic out of manager into its
 547 # own class or module. Consider grouping it with the chunking logic
 548 # in prepare_lists as well.
 549 if num_workers == 1:
 550 return self._shard_in_two(test_files)
 551 elif fully_parallel:
 552 return self._shard_every_file(test_files)
 553 else:
 554 return self._shard_by_directory(test_files, num_workers)
 555
 556 def _shard_in_two(self, test_files):
 557 """Returns two shards, one with all the tests requiring a lock and one with the rest.
 558
 559 This is used when there's only one worker, to minimize the per-shard overhead."""
 560 locked_inputs = []
 561 unlocked_inputs = []
 562 for test_file in test_files:
 563 test_input = self._get_test_input_for_file(test_file)
 564 if self._test_requires_lock(test_file):
 565 locked_inputs.append(test_input)
 566 else:
 567 unlocked_inputs.append(test_input)
 568 return ([('locked_tests', locked_inputs)], [('unlocked_tests', unlocked_inputs)])
 569
 570 def _shard_every_file(self, test_files):
 571 """Returns two lists of shards, each shard containing a single test file.
 572
 573 This mode gets maximal parallelism at the cost of much higher flakiness."""
549574 locked_shards = []
550575 unlocked_shards = []
551  tests_to_http_lock = []
552  if not use_real_shards:
553  for test_file in test_files:
554  test_input = self._get_test_input_for_file(test_file)
555  if self._test_requires_lock(test_file):
556  tests_to_http_lock.append(test_input)
557  else:
558  unlocked_shards.append((".", [test_input]))
559  else:
560  tests_by_dir = {}
561  for test_file in test_files:
562  directory = self._get_dir_for_test_file(test_file)
563  test_input = self._get_test_input_for_file(test_file)
564  if self._test_requires_lock(test_file):
565  tests_to_http_lock.append(test_input)
566  else:
567  tests_by_dir.setdefault(directory, [])
568  tests_by_dir[directory].append(test_input)
569  for directory in tests_by_dir:
570  test_list = tests_by_dir[directory]
571  test_list_tuple = (directory, test_list)
572  unlocked_shards.append(test_list_tuple)
 576 for test_file in test_files:
 577 test_input = self._get_test_input_for_file(test_file)
 578
 579 # Note that we use a '.' for the shard name; the name doesn't really
 580 # matter, and the only other meaningful value would be the filename,
 581 # which would be really redundant.
 582 if self._test_requires_lock(test_file):
 583 locked_shards.append(('.', [test_input]))
 584 else:
 585 unlocked_shards.append(('.', [test_input]))
573586
574  # Sort the shards by directory name.
575  unlocked_shards.sort(lambda a, b: cmp(a[0], b[0]))
 587 return locked_shards, unlocked_shards
576588
577  if tests_to_http_lock:
578  locked_shards = [("tests_to_http_lock", tests_to_http_lock)]
 589 def _shard_by_directory(self, test_files, num_workers):
 590 """Returns two lists of shards, each shard containing all the files in a directory.
579591
580  return (locked_shards, unlocked_shards)
 592 This is the default mode, and gets as much parallelism as we can while
 593 minimizing flakiness caused by inter-test dependencies."""
 594 locked_shards = []
 595 unlocked_shards = []
 596 tests_by_dir = {}
 597 # FIXME: Given that the tests are already sorted by directory,
 598 # we can probably rewrite this to be clearer and faster.
 599 for test_file in test_files:
 600 directory = self._get_dir_for_test_file(test_file)
 601 test_input = self._get_test_input_for_file(test_file)
 602 tests_by_dir.setdefault(directory, [])
 603 tests_by_dir[directory].append(test_input)
 604
 605 for directory in tests_by_dir:
 606 test_list = tests_by_dir[directory]
 607 test_list_tuple = (directory, test_list)
 608 if self._test_requires_lock(directory):
 609 locked_shards.append(test_list_tuple)
 610 else:
 611 unlocked_shards.append(test_list_tuple)
 612
 613 # Sort the shards by directory name.
 614 locked_shards.sort(lambda a, b: cmp(a[0], b[0]))
 615 unlocked_shards.sort(lambda a, b: cmp(a[0], b[0]))
 616
 617 return (self._resize_shards(locked_shards, self._max_locked_shards(num_workers)), unlocked_shards)
 618
 619 def _max_locked_shards(self, num_workers):
 620 # Put a ceiling on the number of locked shards, so that we
 621 # don't hammer the servers too badly.
 622
 623 # FIXME: For now, limit to one shard. After testing to make sure we
 624 # can handle multiple shards, we should probably do something like
 625 # limit this to no more than a quarter of all workers, e.g.:
 626 # return max(math.ceil(num_workers / 4.0), 1)
 627 return 1
 628
 629 def _resize_shards(self, shards, max_shards):
 630 dirs_per_new_shard = math.ceil(len(shards) / max_shards * 1.0)
 631 new_shards = []
 632 i = 1
 633 j = 1
 634 tests = []
 635 while shards:
 636 shard = shards.pop()
 637 tests.extend(shard[1])
 638 i += 1
 639 if i > dirs_per_new_shard:
 640 new_shards.append(('locked_shard_%d' % j, tests))
 641 i = 0
 642 j += 1
 643 tests = []
 644 if tests:
 645 new_shards.append(('locked_shard_%d' % j, tests))
 646 return new_shards
581647
582648 def _contains_tests(self, subdir):
583649 for test_file in self._test_files:

@@class Manager:
585651 return True
586652 return False
587653
588  def _num_workers(self, num_shards):
589  num_workers = min(int(self._options.child_processes), num_shards)
 654 def _log_num_workers(self, num_workers, num_shards, num_locked_shards):
590655 driver_name = self._port.driver_name()
591656 if num_workers == 1:
592657 self._printer.print_config("Running 1 %s over %s" %
593658 (driver_name, grammar.pluralize('shard', num_shards)))
594659 else:
595  self._printer.print_config("Running %d %ss in parallel over %d shards" %
596  (num_workers, driver_name, num_shards))
597  return num_workers
 660 self._printer.print_config("Running %d %ss in parallel over %d shards (%d locked)" %
 661 (num_workers, driver_name, num_shards, num_locked_shards))
598662
599663 def _run_tests(self, file_list, result_summary):
600664 """Runs the tests in the file_list.

@@class Manager:
623687
624688 self._printer.print_update('Sharding tests ...')
625689 locked_shards, unlocked_shards = self._shard_tests(file_list,
626  int(self._options.child_processes) > 1 and not self._options.experimental_fully_parallel)
 690 int(self._options.child_processes), self._options.experimental_fully_parallel)
627691
628692 # FIXME: We don't have a good way to coordinate the workers so that
629693 # they don't try to run the shards that need a lock if we don't actually

@@class Manager:
638702 if locked_shards:
639703 self.start_servers_with_lock()
640704
641  num_workers = self._num_workers(len(all_shards))
 705 num_workers = min(int(self._options.child_processes), len(all_shards))
 706 self._log_num_workers(num_workers, len(all_shards), len(locked_shards))
 707
642708 manager_connection = manager_worker_broker.get(self._port, self._options,
643709 self, worker.Worker)
644710

Tools/Scripts/webkitpy/layout_tests/layout_package/manager_unittest.py

@@class ManagerWrapper(manager.Manager):
4343 return test_file
4444
4545
46 class ManagerTest(unittest.TestCase):
47  def test_shard_tests(self):
48  # Test that _shard_tests in test_runner.TestRunner really
49  # put the http tests first in the queue.
 46class ShardingTests(unittest.TestCase):
 47 test_list = [
 48 "LayoutTests/http/tests/websocket/tests/unicode.htm",
 49 "LayoutTests/animations/keyframes.html",
 50 "LayoutTests/http/tests/security/view-source-no-refresh.html",
 51 "LayoutTests/http/tests/websocket/tests/websocket-protocol-ignored.html",
 52 "LayoutTests/fast/css/display-none-inline-style-change-crash.html",
 53 "LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html",
 54 "LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html",
 55 "LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html",
 56 "LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html",
 57 ]
 58
 59 def get_shards(self, num_workers, fully_parallel):
5060 port = Mock()
5161 port._filesystem = filesystem_mock.MockFileSystem()
52  manager = ManagerWrapper(port=port, options=Mock(), printer=Mock())
53 
54  test_list = [
55  "LayoutTests/websocket/tests/unicode.htm",
56  "LayoutTests/animations/keyframes.html",
57  "LayoutTests/http/tests/security/view-source-no-refresh.html",
58  "LayoutTests/websocket/tests/websocket-protocol-ignored.html",
59  "LayoutTests/fast/css/display-none-inline-style-change-crash.html",
60  "LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html",
61  "LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html",
62  "LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html",
63  "LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html",
64  ]
65 
66  expected_tests_to_http_lock = set([
67  'LayoutTests/websocket/tests/unicode.htm',
68  'LayoutTests/http/tests/security/view-source-no-refresh.html',
69  'LayoutTests/websocket/tests/websocket-protocol-ignored.html',
70  'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html',
71  ])
72 
73  single_locked, single_unlocked = manager._shard_tests(test_list, False)
74  multi_locked, multi_unlocked = manager._shard_tests(test_list, True)
75 
76  self.assertEqual("tests_to_http_lock", single_locked[0][0])
77  self.assertEqual(expected_tests_to_http_lock, set(single_locked[0][1]))
78  self.assertEqual("tests_to_http_lock", multi_locked[0][0])
79  self.assertEqual(expected_tests_to_http_lock, set(multi_locked[0][1]))
 62 self.manager = ManagerWrapper(port=port, options=Mock(), printer=Mock())
 63 return self.manager._shard_tests(self.test_list, num_workers, fully_parallel)
 64
 65 def test_shard_by_dir(self):
 66 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False)
 67
 68 # Note that although there are tests in multiple dirs that need locks,
 69 # they are crammed into a single shard in order to reduce the # of
 70 # workers hitting the server at once.
 71 self.assertEquals(locked,
 72 [('locked_shard_1',
 73 ['LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html',
 74 'LayoutTests/http/tests/websocket/tests/unicode.htm',
 75 'LayoutTests/http/tests/websocket/tests/websocket-protocol-ignored.html',
 76 'LayoutTests/http/tests/security/view-source-no-refresh.html'])])
 77 self.assertEquals(unlocked,
 78 [('animations',
 79 ['LayoutTests/animations/keyframes.html']),
 80 ('dom/html/level2/html',
 81 ['LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html',
 82 'LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html']),
 83 ('fast/css',
 84 ['LayoutTests/fast/css/display-none-inline-style-change-crash.html']),
 85 ('ietestcenter/Javascript',
 86 ['LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
 87
 88 def test_shard_every_file(self):
 89 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True)
 90 self.assertEquals(locked,
 91 [('.', ['LayoutTests/http/tests/websocket/tests/unicode.htm']),
 92 ('.', ['LayoutTests/http/tests/security/view-source-no-refresh.html']),
 93 ('.', ['LayoutTests/http/tests/websocket/tests/websocket-protocol-ignored.html']),
 94 ('.', ['LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html'])])
 95 self.assertEquals(unlocked,
 96 [('.', ['LayoutTests/animations/keyframes.html']),
 97 ('.', ['LayoutTests/fast/css/display-none-inline-style-change-crash.html']),
 98 ('.', ['LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html']),
 99 ('.', ['LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
 100 ('.', ['LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html'])])
 101
 102 def test_shard_in_two(self):
 103 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False)
 104 self.assertEquals(locked,
 105 [('locked_tests',
 106 ['LayoutTests/http/tests/websocket/tests/unicode.htm',
 107 'LayoutTests/http/tests/security/view-source-no-refresh.html',
 108 'LayoutTests/http/tests/websocket/tests/websocket-protocol-ignored.html',
 109 'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html'])])
 110 self.assertEquals(unlocked,
 111 [('unlocked_tests',
 112 ['LayoutTests/animations/keyframes.html',
 113 'LayoutTests/fast/css/display-none-inline-style-change-crash.html',
 114 'LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html',
 115 'LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html',
 116 'LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html'])])
80117
81118
82119class NaturalCompareTest(unittest.TestCase):