Skip to content

Generate consistent history IDs from final test parameters #1323

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
baev wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Generate consistent history IDs from final test parameters #1323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions allure-citrus/src/main/java/io/qameta/allure/citrus/AllureCitrus.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import static io.qameta.allure.util.ResultsUtils.createThreadLabel;
import static io.qameta.allure.util.ResultsUtils.createTitlePath;
import static io.qameta.allure.util.ResultsUtils.getProvidedLabels;
import static io.qameta.allure.util.ResultsUtils.md5;

/**
* Reports Citrus test execution to Allure.
Expand Down Expand Up @@ -187,7 +188,10 @@ public void onTestFailure(final TestCase test, final Throwable cause) {
*/
@Override
public void onTestSkipped(final TestCase test) {
//do nothing
if (!isTestStarted(test)) {
startTest(test);
}
stopTest(test, Status.SKIPPED, null);
}

/**
Expand Down Expand Up @@ -217,9 +221,15 @@ public void onTestActionSkipped(final TestCase testCase, final TestAction testAc
private void startTest(final TestCase testCase) {
final AllureExternalKey testKey = createTestKey(testCase);
final Optional<? extends Class<?>> testClass = Optional.ofNullable(testCase.getTestClass());
final String fullName = testClass
.map(Class::getName)
.map(className -> className + "." + testCase.getName())
.orElseGet(testCase::getName);

final TestResult result = new TestResult()
.setName(testCase.getName())
.setFullName(fullName)
.setTestCaseId(md5(fullName))
.setTitlePath(
testClass
.map(ResultsUtils::createTitlePathFromJavaClass)
Expand Down Expand Up @@ -267,7 +277,7 @@ private void stopTest(final TestCase testCase,
.collect(Collectors.toList());

getLifecycle().updateTest(testKey, result -> {
result.setParameters(parameters);
result.getParameters().addAll(parameters);
result.setStatus(status);
result.setStatusDetails(details);
});
Expand All @@ -286,6 +296,15 @@ private AllureExternalKey createTestKey(final TestCase testCase) {
return testKey;
}

private boolean isTestStarted(final TestCase testCase) {
try {
lock.readLock().lock();
return testKeys.containsKey(testCase);
} finally {
lock.readLock().unlock();
}
}

private AllureExternalKey removeTestKey(final TestCase testCase) {
try {
lock.writeLock().lock();
Expand Down
57 changes: 57 additions & 0 deletions allure-citrus/src/test/java/io/qameta/allure/citrus/AllureCitrusTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.consol.citrus.Citrus;
import com.consol.citrus.CitrusContext;
import com.consol.citrus.TestCase;
import com.consol.citrus.TestCaseMetaInfo;
import com.consol.citrus.actions.AbstractTestAction;
import com.consol.citrus.actions.FailAction;
import com.consol.citrus.context.TestContext;
Expand All @@ -27,6 +28,7 @@
import io.qameta.allure.AllureLifecycle;
import io.qameta.allure.Step;
import io.qameta.allure.model.Parameter;
import io.qameta.allure.model.Stage;
import io.qameta.allure.model.Status;
import io.qameta.allure.model.StatusDetails;
import io.qameta.allure.model.StepResult;
Expand All @@ -40,6 +42,7 @@

import java.time.Instant;

import static io.qameta.allure.util.ResultsUtils.md5;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.tuple;
@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -203,6 +206,60 @@ void shouldSetParameters() {
);
}

@AllureFeatures.History
@AllureFeatures.Parameters
@Test
void shouldCalculateIdsFromFinalNativeAndRuntimeParameters() {
final DefaultTestDesigner designer = new DefaultTestDesigner();
designer.name("Runtime parameters");
designer.variable("native", "example");
designer.action(new AbstractTestAction() {
@Override
public void doExecute(final TestContext context) {
Allure.parameter("runtime", "value");
Allure.parameter("excluded", "ignored", true);
}
});

final AllureResults results = run(designer);
final TestResult testResult = results.getTestResults().get(0);
assertThat(testResult.getParameters())
.extracting(Parameter::getName, Parameter::getValue, Parameter::getExcluded)
.containsExactlyInAnyOrder(
tuple("native", "example", null),
tuple("runtime", "value", null),
tuple("excluded", "ignored", true)
);

final String fullName = DefaultTestDesigner.class.getName() + ".Runtime parameters";
assertThat(testResult.getTestCaseId())
.isEqualTo(md5(fullName));
assertThat(testResult.getHistoryId())
.isEqualTo(md5(md5(fullName) + "native" + "example" + "runtime" + "value"));
}

@AllureFeatures.SkippedTests
@AllureFeatures.History
@Test
void shouldReportDisabledTestsWithIds() {
final DefaultTestDesigner designer = new DefaultTestDesigner();
designer.name("Disabled test");
designer.variable("native", "value");
designer.status(TestCaseMetaInfo.Status.DISABLED);

final AllureResults results = run(designer);
final String fullName = DefaultTestDesigner.class.getName() + ".Disabled test";
final String testCaseId = md5(fullName);
assertThat(results.getTestResults())
.singleElement()
.satisfies(result -> {
assertThat(result.getStatus()).isEqualTo(Status.SKIPPED);
assertThat(result.getStage()).isEqualTo(Stage.FINISHED);
assertThat(result.getTestCaseId()).isEqualTo(testCaseId);
assertThat(result.getHistoryId()).isEqualTo(md5(testCaseId + "native" + "value"));
});
}

@Step("Run test case {testDesigner}")
private AllureResults run(final TestDesigner testDesigner) {
// a failing citrus test is a valid outcome under test — only fail the harness when
Expand Down
16 changes: 4 additions & 12 deletions allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,11 +178,7 @@ private void handleTestCaseStarted(final TestCaseStarted event) {
// the same way full name is generated for
// org.junit.platform.engine.support.descriptor.ClasspathResourceSource
// to support io.qameta.allure.junitplatform.AllurePostDiscoveryFilter
final String fullName = String.format(
"%s:%d",
getTestCaseUri(testCase),
testCase.getLocation().getLine()
);
final String fullName = getTestCaseLocation(testCase);

final String testCaseUuid = testCase.getId().toString();

Expand All @@ -192,7 +188,6 @@ private void handleTestCaseStarted(final TestCaseStarted event) {
final TestResult result = new TestResult()
.setUuid(testCaseUuid)
.setTestCaseId(getTestCaseId(testCase))
.setHistoryId(getHistoryId(testCase))
.setFullName(fullName)
.setTitlePath(titlePath)
.setName(name)
Expand All @@ -205,9 +200,7 @@ private void handleTestCaseStarted(final TestCaseStarted event) {
);

if (scenarioDefinition.getExamples() != null) {
result.setParameters(
getExamplesAsParameters(scenarioDefinition, testCase)
);
result.getParameters().addAll(getExamplesAsParameters(scenarioDefinition, testCase));
}

final String description = Stream.of(feature.getDescription(), scenarioDefinition.getDescription())
Expand Down Expand Up @@ -364,9 +357,8 @@ private void handleEmbedEvent(final EmbedEvent event) {
);
}

private String getHistoryId(final TestCase testCase) {
final String testCaseLocation = getTestCaseUri(testCase) + COLON + testCase.getLocation().getLine();
return md5(testCaseLocation);
private String getTestCaseLocation(final TestCase testCase) {
return getTestCaseUri(testCase) + COLON + testCase.getLocation().getLine();
}

private String getTestCaseId(final TestCase testCase) {
Expand Down
42 changes: 40 additions & 2 deletions allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureCucumber7JvmTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ void shouldPersistHistoryIdForScenarios() {

final List<TestResult> testResults = results.getTestResults();
assertThat(testResults.get(0).getHistoryId())
.isEqualTo("892e5eabe51184301cf1358453c9f052");
.isEqualTo(md5(md5("src/test/resources/features/simple.feature:Add a to b")));
}

@AllureFeatures.History
Expand All @@ -550,7 +550,16 @@ void shouldPersistHistoryIdForExamples() {
final List<TestResult> testResults = results.getTestResults();
assertThat(testResults)
.extracting(TestResult::getHistoryId)
.containsExactlyInAnyOrder("c0f824814a130048e9f86358363cf23e", "646aca5d0775cd4f13161e1ea1a68c39");
.containsExactlyInAnyOrder(
md5(
md5("src/test/resources/features/examples.feature:Scenario with Positive Examples")
+ "a1b3result4"
),
md5(
md5("src/test/resources/features/examples.feature:Scenario with Positive Examples")
+ "a2b4result6"
)
);
}

@AllureFeatures.History
Expand Down Expand Up @@ -782,6 +791,35 @@ void shouldSupportRuntimeApiInStepsWhenHooksAreUsed() {
);
}

@AllureFeatures.History
@AllureFeatures.Parameters
@Test
void shouldCalculateIdsFromRuntimeParametersAtTestEnd() {
final AllureResults results = runFeature("features/runtimeapi.feature");

final TestResult testResult = results.getTestResults().get(0);
assertThat(testResult.getParameters())
.extracting(Parameter::getName, Parameter::getValue, Parameter::getExcluded)
.containsExactlyInAnyOrder(
tuple("runtime", "value", null),
tuple("excluded", "ignored", true)
);
assertThat(testResult.getTestCaseId())
.isEqualTo(
md5(
"src/test/resources/features/runtimeapi.feature:Scenario with Runtime API usage"
)
);
assertThat(testResult.getHistoryId())
.isEqualTo(
md5(
md5(
"src/test/resources/features/runtimeapi.feature:Scenario with Runtime API usage"
) + "runtimevalue"
)
);
}

@SystemProperty(
name = "cucumber.junit-platform.naming-strategy",
value = "long"
Expand Down
2 changes: 2 additions & 0 deletions ...re-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/samples/RuntimeApiSteps.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public void beforeFeature() {

@When("^step 1$")
public void step1() {
Allure.parameter("runtime", "value");
Allure.parameter("excluded", "ignored", true);
Allure.step("step1 nested");
Allure.link("step1", "https://example.org/step1");
Allure.getLifecycle().getCurrentRootKey().ifPresent(key -> {
Expand Down
30 changes: 30 additions & 0 deletions allure-java-commons-test/src/main/java/io/qameta/allure/test/AllureTestCommonsUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,15 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.stream.Stream;

import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_DEFAULT;
import static com.fasterxml.jackson.databind.MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME;
import static io.qameta.allure.util.ResultsUtils.md5;

/**
* Provides utility methods for Allure Java test support support.
Expand Down Expand Up @@ -105,6 +110,31 @@ public static void attach(final AllureResults allureResults) {
);
}

/**
* Calculates the compatibility history id expected in adapter tests.
*
* @param testCaseId the test case id
* @param parameters the final parameters
* @return the expected history id
*/
public static String expectedHistoryId(final String testCaseId, final List<Parameter> parameters) {
final StringBuilder source = new StringBuilder(testCaseId);
final Stream<Parameter> parameterStream = Objects.isNull(parameters) ? Stream.empty() : parameters.stream();
parameterStream
.filter(Objects::nonNull)
.filter(parameter -> !Boolean.TRUE.equals(parameter.getExcluded()))
.sorted(
Comparator.comparing((Parameter parameter) -> Objects.toString(parameter.getName(), ""))
.thenComparing(parameter -> Objects.toString(parameter.getValue(), ""))
)
.forEachOrdered(
parameter -> source
.append(Objects.toString(parameter.getName(), ""))
.append(Objects.toString(parameter.getValue(), ""))
);
return md5(source.toString());
}

private static AttachmentOptions attachmentOptions(final String fileName) {
if (fileName.endsWith(DOT + JSON_EXTENSION) || fileName.endsWith(DOT + TEXT_EXTENSION)) {
return AttachmentOptions.empty();
Expand Down
34 changes: 32 additions & 2 deletions allure-java-commons/src/main/java/io/qameta/allure/AllureLifecycle.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import io.qameta.allure.listener.TestLifecycleListener;
import io.qameta.allure.model.Attachment;
import io.qameta.allure.model.FixtureResult;
import io.qameta.allure.model.Parameter;
import io.qameta.allure.model.ScopeFixtureResult;
import io.qameta.allure.model.ScopeFixtureType;
import io.qameta.allure.model.ScopeResult;
Expand All @@ -45,6 +46,7 @@
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
Expand All @@ -59,11 +61,13 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.stream.Stream;

import static io.qameta.allure.AllureConstants.ATTACHMENT_FILE_SUFFIX;
import static io.qameta.allure.util.ResultsUtils.firstNonEmpty;
import static io.qameta.allure.util.ResultsUtils.getStatus;
import static io.qameta.allure.util.ResultsUtils.getStatusDetails;
import static io.qameta.allure.util.ResultsUtils.md5;
import static io.qameta.allure.util.ServiceLoaderUtils.load;

/**
Expand Down Expand Up @@ -325,8 +329,10 @@ public void updateTest(final Consumer<TestResult> update) {
}

/**
* Stops test by given key. The test must be running; scope metadata is merged into the test here. Unbinds the
* calling thread only if the test is the calling thread's root.
* Stops test by given key. The test must be running; scope metadata is merged into the test here. If the test has
* a test case id but no history id, a compatibility history id is generated from the test case id and the final
* parameters. A history id supplied by a {@link TestLifecycleListener#beforeTestStop(TestResult)} listener is
* preserved. Unbinds the calling thread only if the test is the calling thread's root.
*
* @param key the external test key
*/
Expand All @@ -347,13 +353,37 @@ public void stopTest(final AllureExternalKey key) {
testResult
.setStage(Stage.FINISHED)
.setStop(System.currentTimeMillis());
if (Objects.isNull(testResult.getParameters())) {
testResult.setParameters(new ArrayList<>());
}
applyScopeMetadata(item);
if (Objects.isNull(testResult.getHistoryId()) && Objects.nonNull(testResult.getTestCaseId())) {
testResult.setHistoryId(calculateHistoryId(testResult.getTestCaseId(), testResult.getParameters()));
}
if (isCurrentRoot(key)) {
threadContext.clear();
}
notifier.afterTestStop(testResult);
}

private static String calculateHistoryId(final String testCaseId, final List<Parameter> parameters) {
final StringBuilder source = new StringBuilder(testCaseId);
final Stream<Parameter> parameterStream = Objects.isNull(parameters) ? Stream.empty() : parameters.stream();
parameterStream
.filter(Objects::nonNull)
.filter(parameter -> !Boolean.TRUE.equals(parameter.getExcluded()))
.sorted(
Comparator.comparing((Parameter parameter) -> Objects.toString(parameter.getName(), ""))
.thenComparing(parameter -> Objects.toString(parameter.getValue(), ""))
)
.forEachOrdered(
parameter -> source
.append(Objects.toString(parameter.getName(), ""))
.append(Objects.toString(parameter.getValue(), ""))
);
return md5(source.toString());
}

/**
* Writes test by given key. Waits for the test's pending async attachments before serializing, so the written
* result file is a completion marker: everything it references exists.
Expand Down
Loading
Loading