Merge pull request #115 from qtomlinson/qt/retry

Add retry in integration tests

tools/integration/lib/fetch.js

@@ -3,6 +3,8 @@
 
 const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args))
 // TODO: remove this once fetch is available in Node
+const retry = require('async-retry')
+const RETRIES = 2
 
 function buildPostOpts(json) {
   return {
@@ -12,13 +14,46 @@ function buildPostOpts(json) {
   }
 }
 
-async function callFetch(url, fetchOpts = {}) {
+async function callFetch(url, fetchOpts) {
+  const response = await fetchResponse(url, fetchOpts)
+  return verifyResponse(response)
+}
+
+async function fetchResponse(url, fetchOpts = {}) {
   console.log(`Calling fetch. URL: ${url}, Options: ${JSON.stringify(fetchOpts)}`)
-  const response = await fetch(url, fetchOpts)
+  return await fetch(url, fetchOpts)
+}
+
+function verifyResponse(response) {
   if (!response.ok) {
-    const { status, statusText } = response
+    const { url, status, statusText } = response
     throw new Error(`Error fetching ${url}: ${status}, ${statusText}`)
   }
   return response
 }
-module.exports = { callFetch, buildPostOpts }
+
+async function withRetry(retrier, opts = {}) {
+  const defaultOpts = {
+    retries: RETRIES,
+    onRetry: (err, iAttempt) => {
+      console.log(`Retry ${iAttempt} failed: ${err}`)
+    }
+  }
+  return await retry((bail, iAttempt) => retrier(bail, iAttempt), { ...defaultOpts, ...opts })
+}
+
+async function fetchWithRetry(fetcher, retryOpts) {
+  const response = await withRetry(async () => {
+    const resp = await fetcher()
+    // retry on 5xx
+    if (resp.status >= 500) verifyResponse(resp)
+    return resp
+  }, retryOpts)
+  return verifyResponse(response)
+}
+
+async function callFetchWithRetry(url, fetchOpts) {
+  return fetchWithRetry(() => fetchResponse(url, fetchOpts))
+}
+
+module.exports = { callFetch, buildPostOpts, callFetchWithRetry, fetchWithRetry }

tools/integration/package.json

@@ -27,5 +27,8 @@
     "node-fetch": "^3.3.2",
     "prettier": "^3.2.5",
     "sinon": "^17.0.1"
+  },
+  "dependencies": {
+    "async-retry": "^1.3.3"
   }
 }

tools/integration/test/integration/e2e-test-service/attachmentTest.js

@@ -1,7 +1,7 @@
 // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const { callFetch } = require('../../../lib/fetch')
+const { callFetchWithRetry: callFetch } = require('../../../lib/fetch')
 const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const { strictEqual } = require('assert')
 

tools/integration/test/integration/e2e-test-service/curationTest.js

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { deepStrictEqual, strictEqual, ok } = require('assert')
-const { callFetch, buildPostOpts } = require('../../../lib/fetch')
+const { callFetchWithRetry: callFetch, buildPostOpts } = require('../../../lib/fetch')
 const { devApiBaseUrl, definition } = require('../testConfig')
 
 describe('Validate curation', function () {

tools/integration/test/integration/e2e-test-service/definitionTest.js

@@ -3,7 +3,7 @@
 
 const { omit, isEqual, pick } = require('lodash')
 const { deepStrictEqual, strictEqual, ok } = require('assert')
-const { callFetch, buildPostOpts } = require('../../../lib/fetch')
+const { callFetchWithRetry: callFetch, buildPostOpts } = require('../../../lib/fetch')
 const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')

tools/integration/test/integration/e2e-test-service/noticeTest.js

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const { deepStrictEqual } = require('assert')
-const { callFetch, buildPostOpts } = require('../../../lib/fetch')
+const { callFetchWithRetry: callFetch, buildPostOpts } = require('../../../lib/fetch')
 const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')

tools/integration/test/integration/e2e-test-service/statsTest.js

@@ -1,7 +1,7 @@
 // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const { callFetch } = require('../../../lib/fetch')
+const { callFetchWithRetry: callFetch } = require('../../../lib/fetch')
 const { devApiBaseUrl, definition } = require('../testConfig')
 const { ok } = require('assert')
 

tools/integration/test/integration/e2e-test-service/statusTest.js

@@ -1,7 +1,7 @@
 // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const { callFetch } = require('../../../lib/fetch')
+const { callFetchWithRetry: callFetch } = require('../../../lib/fetch')
 const { devApiBaseUrl, definition } = require('../testConfig')
 const { ok } = require('assert')
 

tools/integration/test/lib/fetchTest.js

@@ -0,0 +1,86 @@
+const { fetchWithRetry } = require('../../lib/fetch')
+const assert = require('assert')
+const sinon = require('sinon')
+const RETRIES = 2
+
+describe('Test retry fetch', function () {
+  it('should try fetching data', async function () {
+    const fetcher = sinon.stub().resolves(new Response('Hello, world!'))
+    const { content } = await test(fetcher)
+    assert.strictEqual(content, 'Hello, world!')
+    assert.strictEqual(fetcher.callCount, 1)
+  })
+
+  it('should try fetching data with retry on exception', async function () {
+    const fetcher = sinon
+      .stub()
+      .onFirstCall()
+      .throws(new Error('Connection error'))
+      .onSecondCall()
+      .resolves(new Response('Hello, world!'))
+    const { content } = await test(fetcher)
+    assert.strictEqual(content, 'Hello, world!')
+    assert.strictEqual(fetcher.callCount, 2)
+  })
+
+  it('should try fetching data with retry on server error', async function () {
+    const fetcher = sinon
+      .stub()
+      .onFirstCall()
+      .resolves(
+        new Response('', {
+          status: 502,
+          statusText: 'Bad Gateway'
+        })
+      )
+      .onSecondCall()
+      .resolves(new Response('Hello, world!'))
+    const { content } = await test(fetcher)
+    assert.strictEqual(content, 'Hello, world!')
+    assert.strictEqual(fetcher.callCount, 2)
+  })
+
+  it('should not retry on client error', async function () {
+    const fetcher = sinon.stub().resolves(
+      new Response('', {
+        status: 400,
+        statusText: 'Bad Request'
+      })
+    )
+    const { errorMessage } = await test(fetcher)
+    assert.ok(errorMessage.includes('Bad Request'))
+    assert.strictEqual(fetcher.callCount, 1)
+  })
+
+  it('should retry until max times', async function () {
+    const fetcher = sinon.stub().rejects(new Error('Connection error'))
+    const { errorMessage } = await test(fetcher)
+    assert.ok(errorMessage.includes('Connection error'))
+    assert.strictEqual(fetcher.callCount, RETRIES + 1)
+  })
+
+  it('should retry until max times with server errors', async function () {
+    const fetcher = sinon.stub().resolves(
+      new Response('', {
+        status: 502,
+        statusText: 'Bad Gateway'
+      })
+    )
+    const { errorMessage } = await test(fetcher)
+    assert.ok(errorMessage.includes('Bad Gateway'))
+    assert.strictEqual(fetcher.callCount, RETRIES + 1)
+  })
+})
+
+async function test(fetcher) {
+  let content = '',
+    errorMessage = ''
+  const retryOpts = { minTimeout: 10, retry: RETRIES }
+  try {
+    const resp = await fetchWithRetry(fetcher, retryOpts)
+    content = await resp.text()
+  } catch (err) {
+    errorMessage = err.message
+  }
+  return { content, errorMessage }
+}