manager.py

@@class Manager:

            Two lists of lists of TestInput objects. The first list should

            only be run under the server lock, the second can be run whenever.

        """

550 # FIXME: We still need to support multiple locked shards.

        locked_shards = []

        unlocked_shards = []

        tests_to_http_lock = []

        if not use_real_shards:

            for test_file in test_files:

555 directory = self._get_dir_for_test_file(test_file)

556556 test_input = self._get_test_input_for_file(test_file)

557557 if self._test_requires_lock(test_file):

558 ~~tests_to_http_~~lock.append(test_input)

558 locked_shards.append((directory, [test_input]))

559559 else:

560 unlocked_shards.append((~~"."~~, [test_input]))

560 unlocked_shards.append((directory, [test_input]))

        else:

            tests_by_dir = {}

            for test_file in test_files:

                directory = self._get_dir_for_test_file(test_file)

                test_input = self._get_test_input_for_file(test_file)

                if self._test_requires_lock(test_file):

                    tests_to_http_lock.append(test_input)

                else:

                    tests_by_dir.setdefault(directory, [])

                    tests_by_dir[directory].append(test_input)

566 tests_by_dir.setdefault(directory, [])

567 tests_by_dir[directory].append(test_input)

            for directory in tests_by_dir:

                test_list = tests_by_dir[directory]

                test_list_tuple = (directory, test_list)

574 unlocked_shards.append(test_list_tuple)

                if self._test_requires_lock(directory):

                    locked_shards.append(test_list_tuple)

                else:

                    unlocked_shards.append(test_list_tuple)

575575

576576 # Sort the shards by directory name.

577 locked_shards.sort(lambda a, b: cmp(a[0], b[0]))

577578 unlocked_shards.sort(lambda a, b: cmp(a[0], b[0]))

578579

579 if tests_to_http_lock:

580 locked_shards.append(("tests_to_http_lock", tests_to_http_lock))

            # Put a ceiling on the number of locked shards, so that we

            # don't hammer the servers too badly. For now we will

            # arbitrarily limit this to no more than a quarter of all workers.

            max_locked_shards = max(math.ceil(int(self._options.child_processes) / 4.0), 1)

            dirs_per_new_shard = math.ceil(len(locked_shards) / max_locked_shards * 1.0)

            new_locked_shards = []

            i = 1

            j = 1

            tests = []

            while locked_shards:

                shard = locked_shards.pop()

                tests.extend(shard[1])

                i += 1

                if i > dirs_per_new_shard:

                    new_locked_shards.append(('locked_shard_%d' % j, tests))

                    i = 0

                    j += 1

                    tests = []

            if tests:

                new_locked_shards.append(('locked_shard_%d' % j, tests))

            locked_shards = new_locked_shards

        return (locked_shards, unlocked_shards)

@@class Manager:

                return True

        return False

590 def _num_workers(self, num_shards):

610 def _num_workers(self, num_shards, num_locked_shards):

        num_workers = min(int(self._options.child_processes), num_shards)

        driver_name = self._port.driver_name()

        if num_workers == 1:

            self._printer.print_config("Running 1 %s over %s" %

                (driver_name, grammar.pluralize('shard', num_shards)))

        else:

597 self._printer.print_config("Running %d %ss in parallel over %d shards" %

598 (num_workers, driver_name, num_shards))

617 self._printer.print_config("Running %d %ss in parallel over %d shards (%d locked)" %

618 (num_workers, driver_name, num_shards, num_locked_shards))

        return num_workers

    def _run_tests(self, file_list, result_summary):

@@class Manager:

        all_shards = self._locked_shards + self._unlocked_shards

635 num_workers = self._num_workers(len(all_shards))

655 num_workers = self._num_workers(len(all_shards), len(self._locked_shards))

        manager_connection = manager_worker_broker.get(self._port, self._options,

                                                       self, worker.Worker)

Tools/Scripts/webkitpy/layout_tests/layout_package/manager_unittest.py

@@class ManagerWrapper(manager.Manager):

class ManagerTest(unittest.TestCase):

    def test_shard_tests(self):

        # FIXME: Should probably replace this with more interesting tests

        # now that the sharding logic is totally different.

        # Test that _shard_tests in test_runner.TestRunner really

        # put the http tests first in the queue.

        port = Mock()

@@class ManagerTest(unittest.TestCase):

        manager = ManagerWrapper(port=port, options=Mock(), printer=Mock())

        test_list = [

55 "LayoutTests/websocket/tests/unicode.htm",

58 "LayoutTests/http/tests/websocket/tests/unicode.htm",

5659 "LayoutTests/animations/keyframes.html",

5760 "LayoutTests/http/tests/security/view-source-no-refresh.html",

58 "LayoutTests/websocket/tests/websocket-protocol-ignored.html",

61 "LayoutTests/http/tests/websocket/tests/websocket-protocol-ignored.html",

          "LayoutTests/fast/css/display-none-inline-style-change-crash.html",

          "LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html",

          "LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html",

@@class ManagerTest(unittest.TestCase):

          "LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html",

        ]

        expected_tests_to_http_lock = set([

          'LayoutTests/websocket/tests/unicode.htm',

          'LayoutTests/http/tests/security/view-source-no-refresh.html',

          'LayoutTests/websocket/tests/websocket-protocol-ignored.html',

          'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html',

        ])

        single_locked, single_unlocked = manager._shard_tests(test_list, False)

        multi_locked, multi_unlocked = manager._shard_tests(test_list, True)

        self.assertEqual("tests_to_http_lock", single_locked[0][0])

        self.assertEqual(expected_tests_to_http_lock, set(single_locked[0][1]))

        self.assertEqual("tests_to_http_lock", multi_locked[0][0])

        self.assertEqual(expected_tests_to_http_lock, set(multi_locked[0][1]))

        self.assertEqual("http/tests/websocket/tests", single_locked[0][0])

        self.assertEqual(set(['LayoutTests/http/tests/websocket/tests/unicode.htm']), set(single_locked[0][1]))

        self.assertEqual("http/tests/security", multi_locked[0][0])

        self.assertEqual(set(['LayoutTests/http/tests/security/view-source-no-refresh.html']), set(multi_locked[0][1]))

class NaturalCompareTest(unittest.TestCase):

Tools/ChangeLog

Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py

Tools/Scripts/webkitpy/layout_tests/layout_package/manager_unittest.py