Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TST: enable timeout config in playwright notebook test #1996

Merged
merged 23 commits into from
Oct 13, 2023
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c176f4e
TST: xfail flaky playwright test
fangchenli Sep 11, 2023
82e7aad
fix typo
fangchenli Sep 11, 2023
2cf7d2f
fix typo
fangchenli Sep 11, 2023
abba893
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 11, 2023
427d960
Merge remote-tracking branch 'upstream/develop' into flaky-playwright…
fangchenli Sep 12, 2023
c662c1d
increase timeout
fangchenli Sep 12, 2023
b594ac7
enable timeout config in notebook test
fangchenli Sep 12, 2023
653ff99
[pre-commit.ci] Apply automatic pre-commit fixes
pre-commit-ci[bot] Sep 12, 2023
df67bf3
fix typing typo
fangchenli Sep 12, 2023
73bee1b
sync param name
fangchenli Sep 12, 2023
4ec13af
Merge remote-tracking branch 'upstream/develop' into flaky-playwright…
fangchenli Sep 12, 2023
c4c7108
Merge remote-tracking branch 'upstream/develop' into flaky-playwright…
fangchenli Sep 13, 2023
ba30c3b
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 13, 2023
eb0e581
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 14, 2023
8c6af70
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 15, 2023
ab4d525
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 19, 2023
fb66ff5
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 19, 2023
4763bc5
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 22, 2023
b06ecfb
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 24, 2023
d45b04d
Merge branch 'develop' into flaky-playwright-test
fangchenli Sep 28, 2023
f6103a4
Merge branch 'develop' into flaky-playwright-test
fangchenli Oct 10, 2023
77366a2
Merge branch 'develop' into flaky-playwright-test
fangchenli Oct 13, 2023
f459867
Merge branch 'develop' into flaky-playwright-test
fangchenli Oct 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 75 additions & 14 deletions tests/common/run_notebook.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import re
import time
from pathlib import Path
from typing import List, Union

from tests.common.navigator import Navigator

Expand All @@ -16,11 +17,13 @@ def __init__(self, navigator: Navigator):
def run(
self,
path,
expected_outputs,
conda_env,
runtime=30000,
retry=2,
exact_match=True,
expected_outputs: List[Union[re.Pattern, str]],
conda_env: str,
timeout: float = 1000,
complition_wait_time: float = 2,
retry: int = 2,
retry_wait_time: float = 5,
exact_match: bool = True,
fangchenli marked this conversation as resolved.
Show resolved Hide resolved
):
"""Run jupyter notebook and check for expected output text anywhere on
the page.
Expand All @@ -30,10 +33,33 @@ def run(

Conda environments may still be being built shortly after deployment.

Parameters
----------
path: str
Path to notebook relative to the root of the jupyterlab instance.
expected_outputs: List[Union[re.Pattern, str]]
Text to look for in the output of the notebook. This can be a
substring of the actual output if exact_match is False.
conda_env: str
Name of conda environment. Python conda environments have the
structure "conda-env-nebari-git-nebari-git-dashboard-py" where
the actual name of the environment is "dashboard".
timeout: float
Time in seconds to wait for the expected output text to appear.
default: 1000
complition_wait_time: float
Time in seconds to wait between checking for expected output text.
default: 2
retry: int
Number of times to retry running the notebook.
default: 2
retry_wait_time: float
Time in seconds to wait between retries.
default: 5
exact_match: bool
If True, the expected output must match exactly. If False, the
expected output must be a substring of the actual output.
default: True
"""
logger.debug(f">>> Running notebook: {path}")
filename = Path(path).name
Expand All @@ -47,11 +73,11 @@ def run(
# make sure that this notebook is one currently selected
self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click()

for i in range(retry):
for _ in range(retry):
self._restart_run_all()
# Wait for a couple of seconds to make sure it's re-started
time.sleep(2)
self._wait_for_commands_completion()
time.sleep(retry_wait_time)
self._wait_for_commands_completion(timeout, complition_wait_time)
all_outputs = self._get_outputs()
self.assert_match_all_outputs(expected_outputs, all_outputs)

Expand Down Expand Up @@ -95,9 +121,32 @@ def open_notebook(self, path):
logger.debug("Path to notebook is invalid")
raise RuntimeError("Path to notebook is invalid")

def assert_code_output(self, code, expected_output):
def assert_code_output(
self,
code: str,
expected_output: List[Union[re.Pattern, str]],
timeout: float = 1000,
complition_wait_time: float = 2,
):
"""
Run code in last cell and check for expected output text anywhere on
the page.


Parameters
----------
code: str
Code to run in last cell.
expected_outputs: List[Union[re.Pattern, str]]
Text to look for in the output of the notebook.
timeout: float
Time in seconds to wait for the expected output text to appear.
default: 1000
complition_wait_time: float
Time in seconds to wait between checking for expected output text.
"""
self.run_in_last_cell(code)
self._wait_for_commands_completion()
self._wait_for_commands_completion(timeout, complition_wait_time)
outputs = self._get_outputs()
self.assert_match_output(expected_output, outputs[-1])

Expand Down Expand Up @@ -125,17 +174,29 @@ def _get_last_cell(self):
return cell
raise ValueError("Unable to get last cell")

def _wait_for_commands_completion(self, timeout=120):
def _wait_for_commands_completion(
self, timeout: float, complition_wait_time: float
):
"""
Wait for commands to finish running

Parameters
----------
timeout: float
Time in seconds to wait for the expected output text to appear.
complition_wait_time: float
Time in seconds to wait between checking for expected output text.
"""
elapsed_time = 0
start_time = time.time()
still_visible = True
start_time = time.time()
while elapsed_time < timeout:
running = self.nav.page.get_by_text("[*]").all()
still_visible = any(list(map(lambda r: r.is_visible(), running)))
elapsed_time = time.time() - start_time
time.sleep(1)
if not still_visible:
break
elapsed_time = time.time() - start_time
time.sleep(complition_wait_time)
if still_visible:
raise ValueError(
f"Timeout Waited for commands to finish, "
Expand Down
2 changes: 1 addition & 1 deletion tests/tests_e2e/playwright/test_playwright.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ def test_notebook(navigator, test_data_root):
path=notebook_name,
expected_outputs=["success: 6"],
conda_env="conda-env-default-py",
runtime=60000,
timeout=500,
)