tests: add firefox e2e tests

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

azure-pipelines.yml

@@ -276,7 +276,7 @@ stages:
               targetPath: 'output-integration/'
               artifact: 'coverage-integration'
               publishLocation: 'pipeline'
-      - job: coverage_e2e
+      - job: coverage_e2e_chrome
         pool:
           vmImage: 'ubuntu-latest'
         steps:
@@ -331,20 +331,90 @@ stages:
             displayName: Prepare unittests and coverage for upload
             inputs:
               script: |
-                mkdir output-e2e
+                mkdir output-e2e-chrome
-                mv unittest.xml output-e2e/unittest.xml
+                mv unittest.xml output-e2e-chrome/unittest.xml
-                mv .coverage output-e2e/coverage
+                mv .coverage output-e2e-chrome/coverage
           - task: PublishPipelineArtifact@1
             condition: failed()
             displayName: Upload screenshots if selenium tests fail
             inputs:
               targetPath: 'selenium_screenshots/'
-              artifact: 'selenium screenshots'
+              artifact: 'chrome selenium screenshots'
               publishLocation: 'pipeline'
           - task: PublishPipelineArtifact@1
             inputs:
-              targetPath: 'output-e2e/'
+              targetPath: 'output-e2e-chrome/'
-              artifact: 'coverage-e2e'
+              artifact: 'coverage-e2e-chrome'
+              publishLocation: 'pipeline'
+      - job: coverage_e2e_firefox
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.9'
+          - task: NodeTool@0
+            inputs:
+              versionSpec: '16.x'
+          - task: DockerCompose@0
+            displayName: Run services
+            inputs:
+              dockerComposeFile: 'scripts/ci.docker-compose.yml'
+              action: 'Run services'
+              buildImages: false
+          - task: CmdLine@2
+            inputs:
+              script: |
+                sudo apt update
+                sudo apt install -y libxmlsec1-dev pkg-config
+                sudo pip install -U wheel pipenv
+                pipenv install --dev --python python3.9
+          - task: DockerCompose@0
+            displayName: Run FirefoxDriver
+            inputs:
+              dockerComposeFile: 'tests/e2e/ci.docker-compose.yml'
+              action: 'Run a specific service'
+              serviceName: 'firefox'
+          - task: CmdLine@2
+            displayName: Build static files for e2e
+            inputs:
+              script: |
+                make gen-web
+                cd web
+                cd api && npm i && cd ..
+                npm i
+                npm run build
+          - task: CmdLine@2
+            displayName: Run full test suite
+            inputs:
+              script: |
+                pipenv run python -m scripts.generate_ci_config
+                pipenv run make test-e2e
+          - task: CmdLine@2
+            condition: always()
+            displayName: Cleanup
+            inputs:
+              script: |
+                docker stop $(docker ps -aq)
+                docker container prune -f
+          - task: CmdLine@2
+            displayName: Prepare unittests and coverage for upload
+            inputs:
+              script: |
+                mkdir output-e2e-firefox
+                mv unittest.xml output-e2e-firefox/unittest.xml
+                mv .coverage output-e2e-firefox/coverage
+          - task: PublishPipelineArtifact@1
+            condition: failed()
+            displayName: Upload screenshots if selenium tests fail
+            inputs:
+              targetPath: 'selenium_screenshots/'
+              artifact: 'firefox selenium screenshots'
+              publishLocation: 'pipeline'
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: 'output-e2e-firefox/'
+              artifact: 'coverage-e2e-firefox'
               publishLocation: 'pipeline'
   - stage: test_combine
     jobs:
@@ -355,8 +425,13 @@ stages:
           - task: DownloadPipelineArtifact@2
             inputs:
               buildType: 'current'
-              artifactName: 'coverage-e2e'
+              artifactName: 'coverage-e2e-firefox'
-              path: "coverage-e2e/"
+              path: "coverage-e2e-firefox/"
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              buildType: 'current'
+              artifactName: 'coverage-e2e-chrome'
+              path: "coverage-e2e-chrome/"
           - task: DownloadPipelineArtifact@2
             inputs:
               buildType: 'current'
@@ -377,7 +452,7 @@ stages:
                 sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
-                pipenv run coverage combine coverage-e2e/coverage coverage-unittest/coverage coverage-integration/coverage
+                pipenv run coverage combine coverage-e2e-chrome/coverage coverage-e2e-firefox/coverage coverage-unittest/coverage coverage-integration/coverage
                 pipenv run coverage xml
                 pipenv run coverage html
           - task: PublishCodeCoverageResults@1
@@ -390,7 +465,8 @@ stages:
             inputs:
               testResultsFormat: 'JUnit'
               testResultsFiles: |
-                coverage-e2e/unittest.xml
+                coverage-e2e-chrome/unittest.xml
+                coverage-e2e-firefox/unittest.xml
                 coverage-integration/unittest.xml
                 coverage-unittest/unittest.xml
               mergeTestResults: true
@@ -403,7 +479,8 @@ stages:
                 npm install -g @zeus-ci/cli
                 npx zeus job update -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -r $BUILD_SOURCEVERSION
                 npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-cobertura+xml" coverage.xml
-                npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e/unittest.xml
+                npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e-firefox/unittest.xml
+                npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-e2e-chrome/unittest.xml
                 npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-integration/unittest.xml
                 npx zeus upload -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -t "application/x-junit+xml" coverage-unittest/unittest.xml
                 npx zeus job update --status=passed -b $BUILD_BUILDID -j $BUILD_BUILDNUMBER -r $BUILD_SOURCEVERSION

tests/e2e/ci.docker-compose.yml

@@ -6,3 +6,8 @@ services:
     volumes:
       - /dev/shm:/dev/shm
     network_mode: host
+  firefox:
+    image: selenium/standalone-firefox:3.141
+    volumes:
+      - /dev/shm:/dev/shm
+    network_mode: host