Flaky Tests: Diagnosis and Cure

// JUnit 5: repeat the suspect test
@RepeatedTest(50)
void appliesCoupon() { /* ... */ }

// Add artificial latency to reveal timing bugs (BiDi, Chromium and Firefox)
// or via CDP on Chromium:
driver.execute_cdp_cmd — see Python; in Java use Network.emulateNetworkConditions from DevTools

// Slow the CPU so renders take longer (CDP)
devTools.send(Emulation.setCPUThrottlingRate(4));

# pytest-repeat: pytest tests/test_coupon.py --count=50 -x
# pytest-randomly: shuffle test order to expose state leakage
# pytest -p randomly --randomly-seed=12345

# Make timing bugs likely: throttle the network and CPU
driver.execute_cdp_cmd("Network.enable", {})
driver.execute_cdp_cmd("Network.emulateNetworkConditions",
                     {"offline": False, "latency": 300, "downloadThroughput": 50_000, "uploadThroughput": 20_000})
driver.execute_cdp_cmd("Emulation.setCPUThrottlingRate", {"rate": 4})

// mocha: --retries is NOT the tool; use a loop in a scratch file
for (let i = 0; i < 50; i++) {
it(`applies coupon (run ${i})`, async () => { /* ... */ });
}

// Throttle to reveal timing bugs (Chromium)
await driver.sendDevToolsCommand('Network.enable', {});
await driver.sendDevToolsCommand('Network.emulateNetworkConditions',
{ offline: false, latency: 300, downloadThroughput: 50000, uploadThroughput: 20000 });
await driver.sendDevToolsCommand('Emulation.setCPUThrottlingRate', { rate: 4 });

// NUnit: [Repeat(50)] on the suspect test
[Test, Repeat(50)]
public void AppliesCoupon() { /* ... */ }

// Throttle via CDP (Chromium)
var session = ((ChromeDriver)driver).GetDevToolsSession();
await session.SendCommand("Network.enable", new JsonObject());
await session.SendCommand("Emulation.setCPUThrottlingRate", JsonSerializer.SerializeToNode(new { rate = 4 }));

// FLAKY: waits for presence, but the element is re-rendered after data loads
WebElement total = wait.until(ExpectedConditions.presenceOfElementLocated(By.id("total")));
driver.findElement(By.id("apply-coupon")).click();
assertEquals("$89.00", total.getText());          // stale, or reads the pre-coupon value

// ROBUST: wait for the state you actually assert on
driver.findElement(By.id("apply-coupon")).click();
wait.until(ExpectedConditions.textToBe(By.id("total"), "$89.00"));

// ROBUST alternative when the value is unknown: wait for it to change, then stabilise
String before = driver.findElement(By.id("total")).getText();
driver.findElement(By.id("apply-coupon")).click();
wait.until(d -> !d.findElement(By.id("total")).getText().equals(before));

# FLAKY
total = wait.until(EC.presence_of_element_located((By.ID, "total")))
driver.find_element(By.ID, "apply-coupon").click()
assert total.text == "$89.00"

# ROBUST
driver.find_element(By.ID, "apply-coupon").click()
wait.until(EC.text_to_be_present_in_element((By.ID, "total"), "$89.00"))

# ROBUST when the value is unknown
before = driver.find_element(By.ID, "total").text
driver.find_element(By.ID, "apply-coupon").click()
wait.until(lambda d: d.find_element(By.ID, "total").text != before)

// FLAKY
const total = await driver.wait(until.elementLocated(By.id('total')), 10000);
await driver.findElement(By.id('apply-coupon')).click();
assert.strictEqual(await total.getText(), '$89.00');

// ROBUST
await driver.findElement(By.id('apply-coupon')).click();
await driver.wait(until.elementTextIs(driver.findElement(By.id('total')), '$89.00'), 10000);

// FLAKY
var total = wait.Until(d => d.FindElement(By.Id("total")));
driver.FindElement(By.Id("apply-coupon")).Click();
Assert.That(total.Text, Is.EqualTo("$89.00"));

// ROBUST
driver.FindElement(By.Id("apply-coupon")).Click();
wait.Until(d => d.FindElement(By.Id("total")).Text == "$89.00");

// JUnit 5
@Tag("flaky")
@Test
void appliesCouponUnderLoad() { /* ... */ }

// Main job:      mvn test -Dgroups='!flaky'
// Quarantine job: mvn test -Dgroups=flaky   (allowed to fail, results tracked)

@pytest.mark.flaky(reason="JIRA-1234 race in coupon service")
def test_applies_coupon_under_load(driver): ...

# Main job:       pytest -m "not flaky"
# Quarantine job: pytest -m flaky --continue-on-collection-errors  (allowed to fail)
# Register the marker in pytest.ini: markers = flaky: known flaky, tracked in JIRA

// mocha grep on a title convention
it('applies coupon under load @flaky', async () => { /* ... */ });

// Main job:       mocha --grep @flaky --invert
// Quarantine job: mocha --grep @flaky

[Test, Category("Flaky")]
public void AppliesCouponUnderLoad() { /* ... */ }

// Main job:       dotnet test --filter "TestCategory!=Flaky"
// Quarantine job: dotnet test --filter "TestCategory=Flaky"