From 8736b31455d52246153b5fada3fd328b5704218c Mon Sep 17 00:00:00 2001 From: mi-ac Date: Fri, 20 Sep 2024 20:11:45 +0200 Subject: [PATCH] Optimize tests-syncer crashtest query (#4267) This limits the crashtest list already when executing the query instead of fetching all and pruning later. This also sorts the tests newest first, so that we rather archive newer tests if the cap is reached. Context: http://b/365801496 --- .../other-bots/chromium-tests-syncer/run.py | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/python/other-bots/chromium-tests-syncer/run.py b/src/python/other-bots/chromium-tests-syncer/run.py index 17068ad9ec..916b7a2c4a 100644 --- a/src/python/other-bots/chromium-tests-syncer/run.py +++ b/src/python/other-bots/chromium-tests-syncer/run.py @@ -48,7 +48,16 @@ def unpack_crash_testcases(crash_testcases_directory): """Unpacks the old crash testcases in the provided directory.""" count = 0 - for testcase in ndb_utils.get_all_from_model(data_types.Testcase): + # Make sure that it is a unique crash testcase. Ignore duplicates, + # uploaded repros. Check if the testcase is fixed. If not, skip. + # Only use testcases that have bugs associated with them. + # Sort latest first. + testcases = data_types.Testcase.query( + data_types.Testcase.status == 'Processed', + ndb_utils.is_false(data_types.Testcase.open), + data_types.Testcase.bug_information != + '').order(-data_types.Testcase.timestamp) + for testcase in testcases: count += 1 if count >= MAX_TESTCASES: logs.info(f'{MAX_TESTCASES} testcases reached.') @@ -62,33 +71,20 @@ def unpack_crash_testcases(crash_testcases_directory): if testcase_id in STORED_TESTCASES_LIST: continue - # 2. Make sure that it is a unique crash testcase. Ignore duplicates, - # uploaded repros. - if testcase.status != 'Processed': - continue - - # 3. Check if the testcase is fixed. If not, skip. - if testcase.open: - continue - - # 4. Check if the testcase has a minimized repro. If not, skip. + # 2. Check if the testcase has a minimized repro. If not, skip. if not testcase.minimized_keys or testcase.minimized_keys == 'NA': continue - # 5. Only use testcases that have bugs associated with them. - if not testcase.bug_information: - continue - - # 6. Existing IPC testcases are un-interesting and unused in further + # 3. Existing IPC testcases are un-interesting and unused in further # mutations. Due to size bloat, ignoring these for now. if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION): continue - # 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests). + # 4. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests). if archive.get_archive_type(testcase.absolute_path): continue - # 8. Skip in-process fuzzer testcases, since these are only applicable to + # 5. Skip in-process fuzzer testcases, since these are only applicable to # fuzz targets and don't run with blackbox binaries. if testcase.fuzzer_name and testcase.fuzzer_name in ENGINE_FUZZER_NAMES: continue