Skip to content

Commit 89b8f12

Browse files
perf2711perf2711
committed
[INT-324] split custom ID tests, add RUN_IT_SINGLE env var
1 parent 4ffcba3 commit 89b8f12

File tree

5 files changed

+119
-64
lines changed

5 files changed

+119
-64
lines changed

‎tests/env-vars-build-id.spec.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ describe('Build ID env var', () => {
6868
expect(build).toBeTruthy();
6969
expect(build?.id).toEqual(buildId);
7070
expect(build?.name).toEqual(SAUCE_VISUAL_BUILD_NAME);
71-
expect(build?.diffs?.nodes.length).toBe(1);
71+
expect(build?.diffs?.nodes.length).toBeGreaterThanOrEqual(1);
7272
expect(build?.status).toBe(BuildStatus.Running);
7373
},
7474
15 * 1000

‎tests/env-vars-custom-id.spec.ts

Lines changed: 10 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
import { BuildStatus, SauceRegion, getApi } from '@saucelabs/visual';
22
import {
33
RE_VISUAL_BUILD_ID,
4-
RE_VISUAL_BUILD_LINK,
54
SAUCE_VISUAL_BUILD_NAME,
6-
waitStatusForBuild,
75
} from './utils/helpers';
86
import { execute } from './utils/process';
97
import { FileHandle } from 'fs/promises';
@@ -18,12 +16,9 @@ const visualApi = getApi({
1816
});
1917

2018
const customId = randomBytes(20).toString('hex');
21-
const customId2 = randomBytes(20).toString('hex');
2219

2320
let fileOutput: FileHandle | undefined;
24-
let dockerOutput = '';
2521
let externalBuildId = '';
26-
let buildId = '';
2722

2823
describe('Custom ID env var', () => {
2924
beforeAll(
@@ -34,7 +29,10 @@ describe('Custom ID env var', () => {
3429
displayOutputOnFailure: true,
3530
pipeOutput: false,
3631
fileOutput,
37-
}
32+
env: {
33+
SAUCE_REGION: region,
34+
},
35+
},
3836
);
3937
expect(result.statusCode).toEqual(0);
4038
const cliOutput = result.stdout;
@@ -52,6 +50,7 @@ describe('Custom ID env var', () => {
5250
`docker run --rm -e SAUCE_USERNAME -e SAUCE_ACCESS_KEY \\
5351
-e SAUCE_VISUAL_BUILD_NAME \\
5452
-e SAUCE_VISUAL_CUSTOM_ID \\
53+
-e SAUCE_REGION \\
5554
${process.env.CONTAINER_IMAGE_NAME}`,
5655
{
5756
displayOutputOnFailure: true,
@@ -60,14 +59,14 @@ describe('Custom ID env var', () => {
6059
env: {
6160
SAUCE_VISUAL_BUILD_NAME: SAUCE_VISUAL_BUILD_NAME,
6261
SAUCE_VISUAL_CUSTOM_ID: customId,
62+
SAUCE_REGION: region,
6363
},
6464
}
6565
);
6666
// Storybook container exits with code 1, this is expected behaviour
6767
if (!process.env.CONTAINER_IMAGE_NAME?.includes('storybook')) {
6868
expect(result.statusCode).toEqual(0);
6969
}
70-
dockerOutput = result.stdout;
7170
},
7271
2 * 60 * 1000
7372
);
@@ -79,66 +78,12 @@ describe('Custom ID env var', () => {
7978
expect(build).toBeTruthy();
8079
expect(build?.id).toEqual(externalBuildId);
8180
expect(build?.name).toEqual(SAUCE_VISUAL_BUILD_NAME);
82-
expect(build?.diffs?.nodes.length).toBe(1);
81+
expect(build?.diffs?.nodes.length).toBeGreaterThanOrEqual(1);
8382
expect(build?.status).toBe(BuildStatus.Running);
8483
},
8584
15 * 1000
8685
);
8786

88-
it(
89-
'runs the docker image with an unlinked custom ID in place',
90-
async () => {
91-
const result = await execute(
92-
`docker run --rm -e SAUCE_USERNAME -e SAUCE_ACCESS_KEY \\
93-
-e SAUCE_VISUAL_BUILD_NAME \\
94-
-e SAUCE_VISUAL_CUSTOM_ID \\
95-
${process.env.CONTAINER_IMAGE_NAME}`,
96-
{
97-
displayOutputOnFailure: true,
98-
pipeOutput: false,
99-
fileOutput,
100-
env: {
101-
SAUCE_VISUAL_BUILD_NAME: SAUCE_VISUAL_BUILD_NAME,
102-
SAUCE_VISUAL_CUSTOM_ID: customId2,
103-
},
104-
}
105-
);
106-
107-
// Storybook container exits with code 1, this is expected behaviour
108-
if (!process.env.CONTAINER_IMAGE_NAME?.includes('storybook')) {
109-
expect(result.statusCode).toEqual(0);
110-
}
111-
dockerOutput = result.stdout;
112-
},
113-
2 * 60 * 1000
114-
);
115-
116-
it('returns a new build ID (not external build ID)', async () => {
117-
expect(dockerOutput.length).toBeGreaterThan(0);
118-
119-
const links = [...dockerOutput.matchAll(RE_VISUAL_BUILD_LINK)];
120-
expect(links.length).toBeGreaterThanOrEqual(1);
121-
buildId = links[0][4];
122-
expect(buildId).not.toEqual(externalBuildId);
123-
});
124-
125-
it(
126-
'build is completed',
127-
async () => {
128-
expect(buildId).toMatch(RE_VISUAL_BUILD_ID);
129-
130-
await waitStatusForBuild(visualApi, buildId, [BuildStatus.Unapproved], {
131-
refreshRate: 1000,
132-
retries: 10,
133-
});
134-
135-
const build = await visualApi.buildWithDiffs(buildId);
136-
expect(build).toBeTruthy();
137-
expect(build?.diffs?.nodes.length).toBe(1);
138-
},
139-
15 * 1000
140-
);
141-
14287
afterAll(
14388
async () => {
14489
const result = await execute(
@@ -147,6 +92,9 @@ describe('Custom ID env var', () => {
14792
displayOutputOnFailure: true,
14893
pipeOutput: false,
14994
fileOutput,
95+
env: {
96+
SAUCE_REGION: region,
97+
},
15098
}
15199
);
152100
expect(result.statusCode).toEqual(0);

‎tests/env-vars-unlinked-custom-id.spec.ts

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import { BuildStatus, SauceRegion, getApi } from '@saucelabs/visual';
2+
import {
3+
RE_VISUAL_BUILD_ID,
4+
RE_VISUAL_BUILD_LINK,
5+
SAUCE_VISUAL_BUILD_NAME,
6+
waitStatusForBuild,
7+
} from './utils/helpers';
8+
import { execute } from './utils/process';
9+
import { FileHandle } from 'fs/promises';
10+
import { randomBytes } from 'crypto';
11+
12+
const region = 'us-west-1' as SauceRegion;
13+
14+
const visualApi = getApi({
15+
region,
16+
user: process.env.SAUCE_USERNAME!,
17+
key: process.env.SAUCE_ACCESS_KEY!,
18+
});
19+
20+
const customId = randomBytes(20).toString('hex');
21+
22+
let fileOutput: FileHandle | undefined;
23+
let dockerOutput = '';
24+
let buildId = '';
25+
26+
describe('Unlinked custom ID env var', () => {
27+
it(
28+
'runs the docker image with an unlinked custom ID in place',
29+
async () => {
30+
const result = await execute(
31+
`docker run --rm -e SAUCE_USERNAME -e SAUCE_ACCESS_KEY \\
32+
-e SAUCE_VISUAL_BUILD_NAME \\
33+
-e SAUCE_VISUAL_CUSTOM_ID \\
34+
-e SAUCE_REGION \\
35+
-e RUN_IT_SINGLE \\
36+
${process.env.CONTAINER_IMAGE_NAME}`,
37+
{
38+
displayOutputOnFailure: true,
39+
pipeOutput: false,
40+
fileOutput,
41+
env: {
42+
SAUCE_VISUAL_BUILD_NAME: SAUCE_VISUAL_BUILD_NAME,
43+
SAUCE_VISUAL_CUSTOM_ID: customId,
44+
SAUCE_REGION: region,
45+
RUN_IT_SINGLE: 'true'
46+
},
47+
}
48+
);
49+
50+
// Storybook container exits with code 1, this is expected behaviour
51+
if (!process.env.CONTAINER_IMAGE_NAME?.includes('storybook')) {
52+
expect(result.statusCode).toEqual(0);
53+
}
54+
dockerOutput = result.stdout;
55+
},
56+
2 * 60 * 1000
57+
);
58+
59+
it('returns a new build ID', async () => {
60+
expect(dockerOutput.length).toBeGreaterThan(0);
61+
62+
const links = [...dockerOutput.matchAll(RE_VISUAL_BUILD_LINK)];
63+
expect(links.length).toBeGreaterThanOrEqual(1);
64+
buildId = links[0][4];
65+
expect(buildId).toMatch(RE_VISUAL_BUILD_ID);
66+
});
67+
68+
it(
69+
'build is completed',
70+
async () => {
71+
expect(buildId).toMatch(RE_VISUAL_BUILD_ID);
72+
73+
await waitStatusForBuild(visualApi, buildId, [BuildStatus.Unapproved], {
74+
refreshRate: 1000,
75+
retries: 30,
76+
});
77+
78+
const build = await visualApi.buildWithDiffs(buildId);
79+
expect(build).toBeTruthy();
80+
expect(build?.diffs?.nodes.length).toBeGreaterThanOrEqual(1);
81+
},
82+
60 * 1000
83+
);
84+
});

‎visual-dotnet/SauceLabs.Visual.IntegrationTests/IntegrationTestAttribute.cs

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,23 @@ namespace SauceLabs.Visual.IntegrationTests;
88
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, Inherited = false)]
99
public class IntegrationTestAttribute : NUnitAttribute, IApplyToTest
1010
{
11+
/// <summary>
12+
/// In some cases, such as passing custom ID when the build does not exist, we only can run a single test class.
13+
/// This is a limitation of the `VisualClient` SDK, which closes a build with or without a custom ID
14+
/// if it was created by the SDK.
15+
/// </summary>
16+
public bool SkipIfSingle { get; }
17+
18+
public IntegrationTestAttribute()
19+
{
20+
SkipIfSingle = false;
21+
}
22+
23+
public IntegrationTestAttribute(bool skipIfSingle)
24+
{
25+
SkipIfSingle = skipIfSingle;
26+
}
27+
1128
public void ApplyToTest(Test test)
1229
{
1330
if (test.RunState == RunState.NotRunnable)
@@ -20,5 +37,11 @@ public void ApplyToTest(Test test)
2037
test.RunState = RunState.Ignored;
2138
test.Properties.Set(PropertyNames.SkipReason, "This test runs only when RUN_IT is \"true\"");
2239
}
40+
41+
if (SkipIfSingle && Environment.GetEnvironmentVariable("RUN_IT_SINGLE") == "true")
42+
{
43+
test.RunState = RunState.Ignored;
44+
test.Properties.Set(PropertyNames.SkipReason, "This test runs only when RUN_IT_SINGLE is NOT \"true\"");
45+
}
2346
}
2447
}

‎visual-dotnet/SauceLabs.Visual.IntegrationTests/LoginPageConcurrent.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
namespace SauceLabs.Visual.IntegrationTests;
77

88
[Parallelizable(ParallelScope.Children)]
9-
[IntegrationTest]
9+
[IntegrationTest(skipIfSingle: true)]
1010
public class LoginPageConcurrent
1111
{
1212
private VisualClient? VisualClient { get; set; }

0 commit comments

Comments
 (0)