Skip to content

Commit 5f080f9

Browse files
abrugaroabrugaro
authored
✨ Automate fixing all issues at once of coolstore repo with low effort and save details (#80)
* initial work Signed-off-by: Alejandro Brugarolas <[email protected]> * changed to aws Signed-off-by: Alejandro Brugarolas <[email protected]> * changed to aws Signed-off-by: Alejandro Brugarolas <[email protected]> * aux Signed-off-by: Alejandro Brugarolas <[email protected]> * aux Signed-off-by: Alejandro Brugarolas <[email protected]> * aux Signed-off-by: Alejandro Brugarolas <[email protected]> * aux Signed-off-by: Alejandro Brugarolas <[email protected]> * aux Signed-off-by: Alejandro Brugarolas <[email protected]> * aux Signed-off-by: Alejandro Brugarolas <[email protected]> * aux Signed-off-by: Alejandro Brugarolas <[email protected]> * increase ec2 instance type for testiong Signed-off-by: Alejandro Brugarolas <[email protected]> * finish Signed-off-by: Alejandro Brugarolas <[email protected]> --------- Signed-off-by: Alejandro Brugarolas <[email protected]>
1 parent 576cb32 commit 5f080f9

9 files changed

+212
-72
lines changed

‎.github/workflows/linux-nightly-ci.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: (Linux) Run Playwright tests nightly on main branch
1+
name: (Linux) Run Playwright tests nightly
22

33
on:
44
schedule:
@@ -75,6 +75,9 @@ jobs:
7575
run: npx playwright test --reporter=list
7676
env:
7777
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
78+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
79+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
80+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
7881
- name: Upload test artifacts
7982
uses: actions/upload-artifact@v4
8083
with:

‎.github/workflows/windows-nightly-ci.yml

+8-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
uses: ./.github/workflows/provision-runner.yml
1212
with:
1313
ec2-image-id: ami-0cf05b9bbda99aed3
14-
ec2-instance-type: m5.2xlarge
14+
ec2-instance-type: m5.4xlarge
1515
security-group-id: sg-0a3e6b53e86d0e69d
1616
subnet-id: subnet-06113672589e7e836
1717
ec2-os-type: windows
@@ -56,12 +56,19 @@ jobs:
5656
echo "set PATH=%PATH%;%nodePath%;%vscodepath%;%JAVA_HOME%\bin;%MAVEN_HOME%\bin" >> execute-tests.bat
5757

5858
echo "set OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> execute-tests.bat
59+
echo "set AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> execute-tests.bat
60+
echo "set AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> execute-tests.bat
61+
echo "set AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }}" >> execute-tests.bat
62+
5963
echo "if exist C:\\Users\\nonadmin\\kai-ci rmdir /s /q C:\\Users\\nonadmin\\kai-ci" >> execute-tests.bat
6064
echo "git clone -b ${{ github.head_ref || github.ref_name }} https://github.com/konveyor/kai-ci C:\\Users\\nonadmin\\kai-ci" >> execute-tests.bat
6165
echo "cd C:\\Users\\nonadmin\\kai-ci" >> execute-tests.bat
6266
echo powershell -Command "\"npm install\"" >> execute-tests.bat
6367
echo "copy .env.example .env" >> execute-tests.bat
6468
echo "echo OPENAI_API_KEY="${{ secrets.OPENAI_API_KEY }}" >> .env" >> execute-tests.bat
69+
echo "echo AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" >> .env" >> execute-tests.bat
70+
echo "echo AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .env" >> execute-tests.bat
71+
echo "echo AWS_DEFAULT_REGION="${{ secrets.AWS_DEFAULT_REGION }}" >> .env" >> execute-tests.bat
6572
echo "echo CI=true >> .env" >> execute-tests.bat
6673
echo "npx playwright test" >> execute-tests.bat
6774
echo "EXIT /B %ERRORLEVEL%" >> execute-tests.bat

‎e2e/pages/application.pages.ts

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import {
2+
_electron as electron,
3+
ElectronApplication,
4+
FrameLocator,
5+
Page,
6+
} from 'playwright';
7+
import { execSync } from 'child_process';
8+
import { downloadLatestKAIPlugin } from '../utilities/download.utils';
9+
import {
10+
cleanupRepo,
11+
getKAIPluginName,
12+
getOSInfo,
13+
getVscodeExecutablePath,
14+
} from '../utilities/utils';
15+
import * as path from 'path';
16+
import { LeftBarItems } from '../enums/left-bar-items.enum';
17+
import { expect } from '@playwright/test';
18+
19+
export class Application {
20+
protected readonly app?: ElectronApplication;
21+
protected readonly window?: Page;
22+
23+
protected constructor(app: ElectronApplication, window: Page) {
24+
this.app = app;
25+
this.window = window;
26+
}
27+
28+
/**
29+
* Returns the main window for further interactions.
30+
*/
31+
public getWindow(): Page {
32+
if (!this.window) {
33+
throw new Error('VSCode window is not initialized.');
34+
}
35+
return this.window;
36+
}
37+
38+
public async pasteContent(content: string) {
39+
await this.app.evaluate(({ clipboard }, content) => {
40+
clipboard.writeText(content);
41+
}, content);
42+
await this.window.keyboard.press('Control+v');
43+
}
44+
45+
public async waitDefault() {
46+
await this.window.waitForTimeout(process.env.CI ? 5000 : 500);
47+
}
48+
}

‎e2e/pages/vscode.pages.ts

+7-48
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,9 @@ import {
1515
import * as path from 'path';
1616
import { LeftBarItems } from '../enums/left-bar-items.enum';
1717
import { expect } from '@playwright/test';
18+
import { Application } from './application.pages';
1819

19-
export class VSCode {
20-
// TODO (@abrugaro) find a better place for constants
21-
public static SCREENSHOTS_FOLDER = 'tests-output/screenshots';
22-
23-
private readonly vscodeApp?: ElectronApplication;
24-
private readonly window?: Page;
25-
26-
private constructor(vscodeApp: ElectronApplication, window: Page) {
27-
this.vscodeApp = vscodeApp;
28-
this.window = window;
29-
}
30-
20+
export class VSCode extends Application {
3121
public static async open(workspaceDir: string) {
3222
const vscodeExecutablePath = getVscodeExecutablePath();
3323
const vscodeApp = await electron.launch({
@@ -113,25 +103,15 @@ export class VSCode {
113103
*/
114104
public async closeVSCode(): Promise<void> {
115105
try {
116-
if (this.vscodeApp) {
117-
await this.vscodeApp.close();
106+
if (this.app) {
107+
await this.app.close();
118108
console.log('VSCode closed successfully.');
119109
}
120110
} catch (error) {
121111
console.error('Error closing VSCode:', error);
122112
}
123113
}
124114

125-
/**
126-
* Returns the main window for further interactions.
127-
*/
128-
public getWindow(): Page {
129-
if (!this.window) {
130-
throw new Error('VSCode window is not initialized.');
131-
}
132-
return this.window;
133-
}
134-
135115
/**
136116
* Iterates through all frames and returns the
137117
* server status panel frame .
@@ -177,12 +157,8 @@ export class VSCode {
177157
const window = this.window;
178158
await this.executeQuickCommand('sources and targets');
179159
await this.waitDefault();
180-
await window.screenshot({
181-
path: `${VSCode.SCREENSHOTS_FOLDER}/debug-target.png`,
182-
});
183160
const targetInput = window.getByPlaceholder('Choose one or more target');
184-
await this.waitDefault();
185-
await expect(targetInput).toBeVisible();
161+
await expect(targetInput).toBeVisible({ timeout: 30000 });
186162
for (const target of targets) {
187163
await targetInput.fill(target);
188164
await this.waitDefault();
@@ -192,7 +168,6 @@ export class VSCode {
192168
.click();
193169
await this.waitDefault();
194170
}
195-
196171
await this.waitDefault();
197172
await targetInput.press('Enter');
198173
await this.waitDefault();
@@ -231,7 +206,7 @@ export class VSCode {
231206
const window = this.getWindow();
232207

233208
const navLi = window.locator(`a[aria-label^="${name}"]`).locator('..');
234-
209+
await expect(navLi).toBeVisible();
235210
if ((await navLi.getAttribute('aria-expanded')) === 'false') {
236211
await navLi.click();
237212
}
@@ -250,10 +225,10 @@ export class VSCode {
250225
public async startServer(): Promise<void> {
251226
await this.openAnalysisView();
252227
const analysisView = await this.getAnalysisIframe();
253-
await this.waitDefault();
254228
if (
255229
!(await analysisView.getByRole('button', { name: 'Stop' }).isVisible())
256230
) {
231+
await analysisView.getByRole('button', { name: 'Start' }).isVisible();
257232
await analysisView.getByRole('button', { name: 'Start' }).click();
258233
await analysisView.getByRole('button', { name: 'Stop' }).isVisible();
259234
}
@@ -314,20 +289,4 @@ export class VSCode {
314289
.getByTitle('Resolution Details')
315290
.contentFrame();
316291
}
317-
318-
// TODO create parent class and move generic functions there
319-
public async pasteContent(content: string) {
320-
await this.vscodeApp.evaluate(({ clipboard }, content) => {
321-
clipboard.writeText(content);
322-
}, content);
323-
await this.window.keyboard.press('Control+v');
324-
}
325-
326-
/**
327-
* Even with Playwright default waiting for actionability, in Electron,
328-
* Playwright tries to perform some actions before the elements are ready to handle those actions
329-
*/
330-
public async waitDefault() {
331-
await this.window.waitForTimeout(process.env.CI ? 5000 : 500);
332-
}
333292
}

‎e2e/tests/global.setup.ts

+23-16
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
import { expect, test, test as setup } from '@playwright/test';
22
import { LeftBarItems } from '../enums/left-bar-items.enum';
33
import { VSCode } from '../pages/vscode.pages';
4-
5-
// TODO : Get repo URL from fixtures
6-
const repoUrl = 'https://github.com/konveyor-ecosystem/coolstore';
4+
import { COOLSTORE_REPO_URL, SCREENSHOTS_FOLDER } from '../utilities/consts';
75

86
setup.describe(
97
'install extension and configure provider settings',
@@ -12,19 +10,19 @@ setup.describe(
1210

1311
test.beforeAll(async () => {
1412
test.setTimeout(1600000);
15-
vscodeApp = await VSCode.init(repoUrl, 'coolstore');
13+
vscodeApp = await VSCode.init(COOLSTORE_REPO_URL, 'coolstore');
1614
});
1715

1816
test.beforeEach(async () => {
19-
// This is for debugging purposes until the Windows tests are stable
17+
const testName = test.info().title.replace(' ', '-');
18+
console.log(`Starting ${testName} at ${new Date()}`);
2019
await vscodeApp.getWindow().screenshot({
21-
path: `${VSCode.SCREENSHOTS_FOLDER}/before-${test.info().title.replace(' ', '-')}.png`,
20+
path: `${SCREENSHOTS_FOLDER}/before-${testName}.png`,
2221
});
2322
});
2423

2524
test('Should open Extensions tab and verify installed extension', async () => {
2625
const window = vscodeApp.getWindow();
27-
await window.waitForTimeout(5000);
2826
await vscodeApp.openLeftBarElement(LeftBarItems.Konveyor);
2927
const heading = window.getByRole('heading', {
3028
name: 'Konveyor',
@@ -33,12 +31,12 @@ setup.describe(
3331
await expect(heading).toBeVisible();
3432
await vscodeApp.getWindow().waitForTimeout(10000);
3533
await window.screenshot({
36-
path: `${VSCode.SCREENSHOTS_FOLDER}/kai-installed-screenshot.png`,
34+
path: `${SCREENSHOTS_FOLDER}/kai-installed-screenshot.png`,
3735
});
3836
});
3937

4038
test('Set Sources and targets', async () => {
41-
await vscodeApp.getWindow().waitForTimeout(5000);
39+
await vscodeApp.waitDefault();
4240
await vscodeApp.selectSourcesAndTargets(
4341
[],
4442
[
@@ -53,20 +51,19 @@ setup.describe(
5351

5452
test('Set Up Konveyor and Start analyzer', async () => {
5553
const window = vscodeApp.getWindow();
56-
await window.waitForTimeout(5000);
5754
await vscodeApp.openSetUpKonveyor();
58-
await window.waitForTimeout(5000);
55+
await vscodeApp.waitDefault();
5956
await window
6057
.getByRole('button', { name: 'Configure Generative AI' })
6158
.click();
62-
await window.waitForTimeout(5000);
59+
await vscodeApp.waitDefault();
6360
await window
6461
.getByRole('button', { name: 'Configure GenAI model settings file' })
6562
.click();
66-
await window.waitForTimeout(5000);
63+
await vscodeApp.waitDefault();
6764

6865
await window.keyboard.press('Control+a+Delete');
69-
await vscodeApp.pasteContent(
66+
/*await vscodeApp.pasteContent(
7067
[
7168
'models:',
7269
' OpenAI: &active',
@@ -77,18 +74,28 @@ setup.describe(
7774
' model: "gpt-4o"',
7875
'active: *active',
7976
].join('\n')
77+
);*/
78+
await vscodeApp.pasteContent(
79+
[
80+
'models:',
81+
' AmazonBedrock: &active',
82+
' provider: "ChatBedrock"',
83+
' args:',
84+
' model_id: "meta.llama3-70b-instruct-v1:0"',
85+
'active: *active',
86+
].join('\n')
8087
);
8188
await window.keyboard.press('Control+s');
8289

83-
await window.waitForTimeout(5000);
90+
await vscodeApp.waitDefault();
8491
await vscodeApp.openSetUpKonveyor();
8592
await window.locator('h3.step-title:text("Open Analysis Panel")').click();
8693
await window
8794
.getByRole('button', { name: 'Open Analysis Panel', exact: true })
8895
.click();
8996
await vscodeApp.startServer();
9097
await vscodeApp.getWindow().screenshot({
91-
path: `${VSCode.SCREENSHOTS_FOLDER}/server-started.png`,
98+
path: `${SCREENSHOTS_FOLDER}/server-started.png`,
9299
});
93100
});
94101
}

‎e2e/tests/global.teardown.ts

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import { test, test as setup } from '@playwright/test';
2+
import * as fs from 'node:fs';
3+
import path from 'path';
4+
import { TESTS_OUTPUT_FOLDER } from '../utilities/consts';
5+
import { getOSInfo } from '../utilities/utils';
6+
import { execSync } from 'child_process';
7+
8+
setup.describe('global teardown', async () => {
9+
test('save coolstore folder as test output and map incidents and files', async () => {
10+
fs.cpSync('coolstore', `${TESTS_OUTPUT_FOLDER}/coolstore`, {
11+
recursive: true,
12+
});
13+
const analysisData = await getFirstAnalysisFileContent();
14+
const incidentsMap = {};
15+
analysisData.forEach((analysis) => {
16+
Object.values(analysis.violations).forEach((violation: any) => {
17+
violation.incidents.forEach((incident) => {
18+
if (!incidentsMap[incident.uri]) {
19+
incidentsMap[incident.uri] = { incidents: [] };
20+
}
21+
incidentsMap[incident.uri].incidents.push(incident);
22+
});
23+
});
24+
});
25+
26+
Object.keys(incidentsMap).forEach((fileUri) => {
27+
incidentsMap[fileUri].updatedContent = fs.readFileSync(
28+
fileUri.replace(getOSInfo() === 'windows' ? 'file:///' : 'file://', ''),
29+
'utf-8'
30+
);
31+
});
32+
33+
/**
34+
* Checkout the repository to get the original files, the modified ones were
35+
* already saved to the tests-output folder
36+
*/
37+
execSync(`cd coolstore && git checkout .`);
38+
39+
Object.keys(incidentsMap).forEach((fileUri) => {
40+
incidentsMap[fileUri].originalContent = fs.readFileSync(
41+
fileUri.replace(getOSInfo() === 'windows' ? 'file:///' : 'file://', ''),
42+
'utf-8'
43+
);
44+
});
45+
46+
fs.writeFileSync(
47+
path.join(TESTS_OUTPUT_FOLDER, 'incidents-map.json'),
48+
JSON.stringify(incidentsMap, null, 2),
49+
'utf-8'
50+
);
51+
});
52+
});
53+
54+
async function getFirstAnalysisFileContent() {
55+
const konveyorFolder = 'coolstore/.vscode/konveyor';
56+
57+
const files = await fs.promises.readdir(konveyorFolder);
58+
59+
const analysisFiles = files.filter((file) =>
60+
/^analysis_\d{8}T\d{6}\.json$/.test(file)
61+
);
62+
63+
if (!analysisFiles.length) {
64+
console.error('No analysis file found.');
65+
return;
66+
}
67+
68+
analysisFiles.sort((a, b) => {
69+
const dateA = a.match(/\d{8}T\d{6}/)?.[0];
70+
const dateB = b.match(/\d{8}T\d{6}/)?.[0];
71+
72+
if (dateA && dateB) {
73+
return dateA.localeCompare(dateB);
74+
}
75+
return 0;
76+
});
77+
78+
const fileContent = await fs.promises.readFile(
79+
path.join(konveyorFolder, analysisFiles[0]),
80+
'utf-8'
81+
);
82+
83+
return JSON.parse(fileContent);
84+
}

0 commit comments

Comments
 (0)