Merge pull request #3096 from gradido/cypress-reporting

feat(other): proper reporting for failing end-to-end tests

.github/workflows/test_e2e.yml

@@ -33,7 +33,6 @@ jobs:
           yarn && yarn dev_reset
           cd ../backend
           yarn && yarn seed
-          cd ..
 
       - name: Boot up test system | docker-compose frontends
         run: docker-compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps frontend admin nginx
@@ -41,18 +40,35 @@ jobs:
       - name: Boot up test system | docker-compose mailserver
         run: docker-compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps mailserver
 
-      - name: Sleep for 15 seconds
-        run: sleep 15s
+      - name: End-to-end tests | prepare
+        run: |
+          wget --no-verbose -O /opt/cucumber-json-formatter "https://github.com/cucumber/json-formatter/releases/download/v19.0.0/cucumber-json-formatter-linux-386"
+          chmod +x /opt/cucumber-json-formatter
+          sudo ln -fs /opt/cucumber-json-formatter /usr/bin/cucumber-json-formatter
+          cd e2e-tests/
+          yarn
 
       - name: End-to-end tests | run tests
         id: e2e-tests
         run: |
           cd e2e-tests/
-          yarn
           yarn run cypress run
-      - name: End-to-end tests | if tests failed, upload screenshots
+
+      - name: End-to-end tests | if tests failed, compile html report
+        if: ${{ failure() && steps.e2e-tests.conclusion == 'failure' }}
+        run: |
+          cd e2e-tests/
+          node create-cucumber-html-report.js
+
+      - name: End-to-end tests | if tests failed, get pr number
+        id: pr
+        if: ${{ failure() && steps.e2e-tests.conclusion == 'failure' }}
+        uses: 8BitJonny/gh-get-current-pr@2.2.0
+
+      - name: End-to-end tests | if tests failed, upload report
+        id: e2e-report
         if: ${{ failure() && steps.e2e-tests.conclusion == 'failure' }}
         uses: actions/upload-artifact@v3
         with:
-          name: cypress-screenshots
-          path: /home/runner/work/gradido/gradido/e2e-tests/cypress/screenshots/
+          name: cypress-report-pr-#${{ steps.pr.outputs.number }}
+          path: /home/runner/work/gradido/gradido/e2e-tests/cypress/reports/cucumber_html_report

e2e-tests/create-cucumber-html-report.js

@@ -0,0 +1,12 @@
+const report = require("multiple-cucumber-html-reporter");
+
+const reportTitle = "Gradido webapp end-to-end test report"
+
+report.generate({
+  jsonDir: "cypress/reports/json_logs",
+  reportPath: "./cypress/reports/cucumber_html_report",
+  pageTitle: reportTitle,
+  reportName: reportTitle,
+  pageFooter: "<div></div>",
+  hideMetadata: true
+});

e2e-tests/cypress.config.ts

@@ -26,14 +26,6 @@ async function setupNodeEvents(
     },
   })
 
-  on('after:run', (results) => {
-    if (results) {
-      // results will be undefined in interactive mode
-      // eslint-disable-next-line no-console
-      console.log(results.status)
-    }
-  })
-
   return config
 }
 

e2e-tests/package.json

@@ -8,9 +8,18 @@
   "license": "Apache-2.0",
   "private": false,
   "cypress-cucumber-preprocessor": {
-    "nonGlobalStepDefinitions": true,
+    "stepDefinitions": "cypress/support/step_definitions/*.ts",
     "json": {
-      "enabled": true
+      "enabled": true,
+      "output": "cypress/reports/json_logs/cucumber_log.json",
+      "formatter": "cucumber-json-formatter"
+    },
+    "messages": {
+      "enabled": true,
+      "output": "cypress/reports/json_logs/messages.ndjson"
+    },
+    "html": {
+      "enabled": false
     }
   },
   "scripts": {
@@ -33,6 +42,7 @@
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-promise": "^6.1.1",
     "jwt-decode": "^3.1.2",
+    "multiple-cucumber-html-reporter": "^3.4.0",
     "prettier": "^2.7.1",
     "typescript": "^4.7.4"
   }