Merge pull request #26 from kudulab/improve-e2e-tests-output

Improve e2e tests output

CHANGELOG.md

@@ -1,6 +1,9 @@
-### 0.10.1 (2020-Oct-18)
+### 0.10.1 (2020-Oct-25)
 
 * Verify if Bash is installed when running Dojo. Dojo performs a shell out and Bash is its dependency. If Bash is not installed, a pretty error will be printed. [#22](https://github.com/kudulab/dojo/issues/22)
+* Use newer pytest version 6.1.1, was 4.4.1
+* Improve e2e tests output - print stdout and stderr on failure
+* Improve e2e tests when checking for warn or error strings
 
 ### 0.10.0 (2020-Sep-06)
 

test/requirements.txt

@@ -1 +1 @@
-pytest==4.4.1
+pytest==6.1.1

test/support/common.py

@@ -36,8 +36,6 @@ def run_dojo_and_set_bash_func(args, env=None):
     return proc
 
 
-def assert_no_warnings_or_errors(text):
-    assert not 'warn' in text
-    assert not 'error' in text
-    assert not 'WARN' in text
-    assert not 'ERROR' in text
+def assert_no_warnings_or_errors(text, full_output_to_print):
+    assert not 'warn' in text.lower(), full_output_to_print
+    assert not 'error' in text.lower(), full_output_to_print

test/test_basics.py

@@ -4,12 +4,14 @@
 
 def test_version_output():
     result = run_dojo(['--version'])
-    assert 'Dojo version' in result.stderr #TODO: (ewa) should be in stdout
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str #TODO: (ewa) should be in stdout
     assert result.returncode == 0
 
 
 def test_help_output():
     result = run_dojo(['--help'])
-    assert 'Usage of dojo' in result.stderr
-    assert 'Driver: docker or docker-compose' in result.stderr
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Usage of dojo' in result.stderr, dojo_combined_output_str
+    assert 'Driver: docker or docker-compose' in result.stderr, dojo_combined_output_str
     assert result.returncode == 0

test/test_docker.py

@@ -8,88 +8,113 @@ def clean_up_docker_container():
 
 def test_docker_container_is_removed():
     result = run_command('docker', ['ps', '-a', '--filter', "name=testdojorunid"])
-    assert not 'testdojorunid' in result.stderr
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert not 'testdojorunid' in result.stderr, dojo_combined_output_str
     assert result.returncode == 0
 
 
 def test_docker_when_zero_exit():
     clean_up_docker_container()
     result = run_dojo('--debug=true --test=true --image=alpine:3.8 whoami'.split(' '))
-    assert 'Dojo version' in result.stderr
-    assert 'root' in result.stdout
-    assert 'alpine:3.8 whoami' in result.stderr
-    assert 'Exit status from run command: 0' in result.stderr
-    assert 'Exit status from cleaning: 0' in result.stderr
-    assert 'Exit status from signals: 0' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'root' in result.stdout, dojo_combined_output_str
+    assert 'alpine:3.8 whoami' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
     test_docker_container_is_removed()
 
 
 def test_docker_capture_output():
     clean_up_docker_container()
+    # run this one test manually: pytest --capture=fd  --verbose test/test_docker.py::test_docker_capture_output
+    # run this without pytest: ./bin/dojo --debug=true --test=true --image=alpine:3.8 sh -c "printenv HOME"
     result = run_dojo(['--debug=true', '--test=true', '--image=alpine:3.8', 'sh', '-c', "printenv HOME"])
-    assert 'Dojo version' in result.stderr
-    assert '/root\n' == result.stdout
-    assert 'Exit status from run command: 0' in result.stderr
-    assert 'Exit status from cleaning: 0' in result.stderr
-    assert 'Exit status from signals: 0' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert '/root\n' == result.stdout, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
     test_docker_container_is_removed()
 
 
+def test_docker_capture_output_when_unable_to_pull_image():
+    clean_up_docker_container()
+    # pytest --capture=fd  --verbose test/test_docker.py::test_docker_capture_output_when_unable_to_pull_image
+    # ./bin/dojo --debug=true --test=true --image=alpine:3.8 sh -c "printenv HOME && hostname"
+    result = run_dojo(['--debug=true', '--test=true', '--image=no_such_image91291925129q783187314218194:abc111aaa.9981412', 'sh', '-c', "printenv HOME"])
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    # this is the main reason for this test:
+    assert 'Unable to find image' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from run command: 125' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
+    assert result.returncode == 125
+    test_docker_container_is_removed()
+
+
 def test_docker_when_non_existent_command():
     clean_up_docker_container()
     result = run_dojo('--debug=true --test=true --image=alpine:3.8 notexistentcommand'.split(' '))
-    assert 'Dojo version' in result.stderr
-    assert 'executable file not found' in result.stderr
-    assert 'Exit status from run command: 127' in result.stderr
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'executable file not found' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from run command: 127' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 127
     test_docker_container_is_removed()
 
 
 def test_docker_when_no_command():
     clean_up_docker_container()
     result = run_dojo('--debug=true --test=true --image=alpine:3.8 -i=false'.split(' '))
-    assert 'Dojo version' in result.stderr
-    assert 'Exit status from run command: 0' in result.stderr
-    assert 'Exit status from cleaning: 0' in result.stderr
-    assert 'Exit status from signals: 0' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
     test_docker_container_is_removed()
 
 
 def test_docker_when_double_dash_command_split():
     clean_up_docker_container()
     result = run_dojo('--debug=true --test=true --image=alpine:3.8 -- whoami'.split(' '))
-    assert 'Dojo version' in result.stderr
-    assert 'root' in result.stdout
-    assert 'alpine:3.8 whoami' in result.stderr
-    assert 'Exit status from run command: 0' in result.stderr
-    assert 'Exit status from cleaning: 0' in result.stderr
-    assert 'Exit status from signals: 0' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'root' in result.stdout, dojo_combined_output_str
+    assert 'alpine:3.8 whoami' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
     test_docker_container_is_removed()
 
 
 def test_docker_when_shell_command():
     clean_up_docker_container()
     result = run_dojo(['--debug=true', '--test=true', '--image=alpine:3.8', 'sh', '-c', 'echo hello'])
-    assert 'Dojo version' in result.stderr
-    assert 'hello' in result.stdout
-    assert 'Exit status from run command: 0' in result.stderr
-    assert 'Exit status from cleaning: 0' in result.stderr
-    assert 'Exit status from signals: 0' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'hello' in result.stdout, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
     test_docker_container_is_removed()
 
@@ -101,11 +126,12 @@ def test_docker_preserves_env_vars():
     result = run_dojo(
         ['--debug=true', '--test=true', '--image=alpine:3.8', 'sh', '-c', 'env | grep ABC'],
         env=envs)
-    assert 'Dojo version' in result.stderr
-    assert 'custom_value' in result.stdout
-    assert 'Exit status from run command: 0' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'custom_value' in result.stdout, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
 
 # Bash experiments:
@@ -147,14 +173,15 @@ def test_docker_preserves_multiline_env_vars():
         # it does not have the Dojo entrypoint.sh.
         ['--debug=true', '--test=true', '--image=alpine:3.8', 'sh', '-c', '"source /etc/dojo.d/variables/00-multiline-vars.sh && env | grep -A 1 ABC"'],
         env=envs)
-    assert 'Dojo version' in result.stderr
-    assert '/etc/dojo.d/variables/00-multiline-vars.sh' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert '/etc/dojo.d/variables/00-multiline-vars.sh' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
-    assert 'Exit status from run command:' in result.stderr
+    assert 'Exit status from run command:' in result.stderr, dojo_combined_output_str
     assert """first line
-second line""" in result.stdout
+second line""" in result.stdout, dojo_combined_output_str
 
 
 def test_docker_preserves_bash_functions_from_env_vars():
@@ -177,29 +204,31 @@ def test_docker_preserves_bash_functions_from_env_vars():
     stdout_value_bytes, stderr_value_bytes = proc.communicate()
     stdout = stdout_value_bytes.decode("utf-8")
     stderr = stderr_value_bytes.decode("utf-8")
-    assert 'Dojo version' in stderr
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(stdout, stderr)
+    assert 'Dojo version' in stderr, dojo_combined_output_str
     # print(stdout)
     # print(stderr)
-    assert 'Written file /tmp/test-dojo-environment-bash-functions-testdojorunid, contents:' in stderr
-    assert 'my_bash_func() {  echo "hello"' in stderr
-    assert '/etc/dojo.d/variables/01-bash-functions.sh' in stderr
-    assert_no_warnings_or_errors(stderr)
-    assert_no_warnings_or_errors(stdout)
+    assert 'Written file /tmp/test-dojo-environment-bash-functions-testdojorunid, contents:' in stderr, dojo_combined_output_str
+    assert 'my_bash_func() {  echo "hello"' in stderr, dojo_combined_output_str
+    assert '/etc/dojo.d/variables/01-bash-functions.sh' in stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(stdout, dojo_combined_output_str)
     # the bash function was invoked
-    assert 'hello' in stdout
-    assert 'Exit status from run command: 0' in stderr
+    assert 'hello' in stdout, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in stderr, dojo_combined_output_str
 
 
 def test_docker_when_custom_relative_directory():
     clean_up_docker_container()
     result = run_dojo(['-c', 'test/test-files/Dojofile', '--debug=true', '--test=true', '--image=alpine:3.8', 'whoami'])
-    assert 'Dojo version' in result.stderr
-    assert 'root' in result.stdout
-    assert 'Exit status from run command: 0' in result.stderr
-    assert 'Exit status from cleaning: 0' in result.stderr
-    assert 'Exit status from signals: 0' in result.stderr
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'root' in result.stdout, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
     assert result.returncode == 0
     test_docker_container_is_removed()
 
@@ -211,21 +240,32 @@ def test_docker_when_nonexistent_custom_relative_directory():
     except FileNotFoundError:
         pass
     result = run_dojo(['-c', 'test/test-files/Dojofile.work_not_exists', '--debug=true', '--test=true', '--image=alpine:3.8', 'whoami'])
-    assert 'Dojo version' in result.stderr
-    assert 'root' in result.stdout
-    assert "test/test-files/not-existent does not exist" in result.stderr
-    assert 'WARN' in result.stderr
-    assert 'Exit status from run command: 0' in result.stderr
-    assert 'Exit status from cleaning: 0' in result.stderr
-    assert 'Exit status from signals: 0' in result.stderr
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert 'Dojo version' in result.stderr, dojo_combined_output_str
+    assert 'root' in result.stdout, dojo_combined_output_str
+    assert "test/test-files/not-existent does not exist" in result.stderr, dojo_combined_output_str
+    assert 'WARN' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from run command: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from cleaning: 0' in result.stderr, dojo_combined_output_str
+    assert 'Exit status from signals: 0' in result.stderr, dojo_combined_output_str
     assert result.returncode == 0
     test_docker_container_is_removed()
 
 
-def test_docker_pull():
+def test_docker_pull_when_image_can_be_pulled():
     result = run_dojo('--debug=true --action=pull --image=alpine:3.8'.split(' '))
-    assert "Dojo version" in result.stderr
-    assert "Pulling from library/alpine" in result.stdout
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert "Dojo version" in result.stderr, dojo_combined_output_str
+    assert "Pulling from library/alpine" in result.stdout, dojo_combined_output_str
     assert result.returncode == 0
-    assert_no_warnings_or_errors(result.stderr)
-    assert_no_warnings_or_errors(result.stdout)
+    assert_no_warnings_or_errors(result.stderr, dojo_combined_output_str)
+    assert_no_warnings_or_errors(result.stdout, dojo_combined_output_str)
+
+
+def test_docker_pull_when_no_such_image_exists():
+    result = run_dojo('--debug=true --action=pull --image=no_such_image91291925129q783187314218194:abc111aaa.9981412'.split(' '))
+    dojo_combined_output_str =  "stdout:\n{0}\nstderror:\n{1}".format(result.stdout, result.stderr)
+    assert "" == result.stdout, dojo_combined_output_str
+    assert "Dojo version" in result.stderr, dojo_combined_output_str
+    assert "repository does not exist or may require 'docker login'" in result.stderr, dojo_combined_output_str
+    assert result.returncode == 1