From de16072171cb60ba1febc2f41f9def3a43622323 Mon Sep 17 00:00:00 2001 From: Rafal Szmit Date: Thu, 24 Aug 2023 15:37:48 +0200 Subject: [PATCH] [AAE-1653] Script to update webdriver locally ADF (#8846) * [AAE-1653] Script to update webdriver locally ADF * update readme * trigger ci --- scripts/webdriver-update-newest/README.md | 37 +++ .../chrome_xml_schema.js | 184 +++++++++++ .../update-to-newest-webdriver.sh | 29 ++ .../webdriver-update-newest/update_schema.js | 304 ++++++++++++++++++ 4 files changed, 554 insertions(+) create mode 100644 scripts/webdriver-update-newest/README.md create mode 100644 scripts/webdriver-update-newest/chrome_xml_schema.js create mode 100755 scripts/webdriver-update-newest/update-to-newest-webdriver.sh create mode 100644 scripts/webdriver-update-newest/update_schema.js diff --git a/scripts/webdriver-update-newest/README.md b/scripts/webdriver-update-newest/README.md new file mode 100644 index 00000000000..80a38ef3d4b --- /dev/null +++ b/scripts/webdriver-update-newest/README.md @@ -0,0 +1,37 @@ +# Run protractor with newest webdriver locally + +## Instruction +To download newest driver simply run script from its directory +`update-to-newest-webdriver.sh` + +Command accepts one parameter to define what OS you are using. By default its set to `mac-x64` +Possible inputs `linux64, mac-arm64, mac-x64, win32, win64` + +Example `./update-to-newest-webdriver.sh win64` - will set driver for windows + +## How it works +1. The script removes your existing driver files from webdriver node_modules +2. Generates two new files (chrome_xml.js and update.js) that have updated methods needed to get the new driver +3. Replaces browser type depending on parameter +4. Copies and replaces the files to the webdriver node_modules +5. Executes command to to update-webdriver using updated code + +## Troubleshooting +If the script fails for any reason. You can do some of these actions manually: +1. Find the two files (chrome_xml.js and update.js) in node_modules/webdriver-manager +2. Replace its contents with (chrome_xml_schema.js and update_schema.js) keep the original names. +3. Change version for specific OS in both files + chrome_xml.js -> ['platform'] == 'mac-x64' e.g. ['platform'] == 'win64' Line 70 + update.js -> 'chromedriver-mac-x64' e.g 'chromedriver-win64' Line 240 +4. Run standard command to update webdriver `./node_modules/webdriver-manager/bin/webdriver-manager update --gecko=false` + + + +## Reason +Latest ChromeDriver Binaries https://googlechromelabs.github.io/chrome-for-testing/ + +Starting with M115 the latest Chrome + ChromeDriver releases per release channel (Stable, Beta, Dev, Canary) are available at the Chrome for Testing availability dashboard. For automated version downloading one can use the convenient JSON endpoints. +The older releases can be found at the Downloads page. + + +Note: Protractor is a depricated tool and this probably won't be fixed. \ No newline at end of file diff --git a/scripts/webdriver-update-newest/chrome_xml_schema.js b/scripts/webdriver-update-newest/chrome_xml_schema.js new file mode 100644 index 00000000000..266eda614b2 --- /dev/null +++ b/scripts/webdriver-update-newest/chrome_xml_schema.js @@ -0,0 +1,184 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const semver = require("semver"); +const config_1 = require("../config"); +const http_utils_1 = require("../http_utils"); +const config_source_1 = require("./config_source"); +class ChromeXml extends config_source_1.XmlConfigSource { + constructor() { + super('chrome', config_1.Config.cdnUrls()['chrome']); + this.maxVersion = config_1.Config.binaryVersions().maxChrome; + } + getUrl(version) { + if (version === 'latest') { + return this.getLatestChromeDriverVersion(); + } + else { + return this.getSpecificChromeDriverVersion(version); + } + } + /** + * Get a list of chrome drivers paths available for the configuration OS type and architecture. + */ + getVersionList() { + return this.getXml().then(xml => { + let versionPaths = []; + let osType = this.getOsTypeName(); + for (let content of xml.ListBucketResult.Contents) { + let contentKey = content.Key[0]; + if ( + // Filter for 32-bit devices, make sure x64 is not an option + (this.osarch.includes('64') || !contentKey.includes('64')) && + // Filter for x86 macs, make sure m1 is not an option + ((this.ostype === 'Darwin' && this.osarch === 'arm64') || !contentKey.includes('m1'))) { + // Filter for only the osType + if (contentKey.includes(osType)) { + versionPaths.push(contentKey); + } + } + } + return versionPaths; + }); + } + /** + * Helper method, gets the ostype and gets the name used by the XML + */ + getOsTypeName() { + // Get the os type name. + if (this.ostype === 'Darwin') { + return 'mac'; + } + else if (this.ostype === 'Windows_NT') { + return 'win'; + } + else { + return 'linux'; + } + } + /** + * Gets the latest item from the XML. + */ + getLatestChromeDriverVersion() { + const path = require('path') + const fs = require('fs') + + const lastKnownGoodVersionsWithDownloads_Url = 'https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json'; + return http_utils_1.requestBody(lastKnownGoodVersionsWithDownloads_Url).then(body => { + const latestVersion_Body = JSON.parse(body)['channels']['Stable'] + + const latestVersion = latestVersion_Body['version'] + const latestVersion_Url = latestVersion_Body['downloads']['chromedriver'].find(obj => obj['platform'] == 'mac-x64')['url'] + + const latestMajorVersion = latestVersion.split('.')[0] + + const localVersion_FileName = fs.readdirSync(path.resolve(__dirname, '..', '..', '..', 'selenium')) + .find(f => f.startsWith(`chromedriver_${latestMajorVersion}`)) || '' + + const localVersion = localVersion_FileName.slice(13, -4) + const localVersion_Url = latestVersion_Url.replace(latestVersion, localVersion) + + const localMajorVersion = localVersion.split('.')[0] + + if (latestMajorVersion == localMajorVersion) { + return Promise.resolve({ + url: localVersion_Url, + version: localVersion, + }) + } else { + return Promise.resolve({ + url: latestVersion_Url, + version: latestVersion, + }) + } + }); + } + /** + * Gets a specific item from the XML. + */ + getSpecificChromeDriverVersion(inputVersion) { + return this.getVersionList().then(list => { + const specificVersion = getValidSemver(inputVersion); + if (specificVersion === '') { + throw new Error(`version ${inputVersion} ChromeDriver does not exist`); + } + let itemFound = ''; + for (let item of list) { + // Get a semantic version. + let version = item.split('/')[0]; + if (semver.valid(version) == null) { + const lookUpVersion = getValidSemver(version); + if (semver.valid(lookUpVersion)) { + // Check to see if the specified version matches. + if (lookUpVersion === specificVersion) { + // When item found is null, check the os arch + // 64-bit version works OR not 64-bit version and the path does not have '64' + if (itemFound == '') { + if (this.osarch === 'x64' || + (this.osarch !== 'x64' && !item.includes(this.getOsTypeName() + '64'))) { + itemFound = item; + } + if (this.osarch === 'arm64' && this.ostype === 'Darwin' && item.includes('m1')) { + itemFound = item; + } + } + else if (this.osarch === 'x64') { + // No win64 version exists, so even on x64 we need to look for win32 + const osTypeNameAndArch = this.getOsTypeName() + (this.getOsTypeName() === 'win' ? '32' : '64'); + if (item.includes(osTypeNameAndArch)) { + itemFound = item; + } + } + } + } + } + } + if (itemFound == '') { + return { url: '', version: inputVersion }; + } + else { + return { url: config_1.Config.cdnUrls().chrome + itemFound, version: inputVersion }; + } + }); + } +} +exports.ChromeXml = ChromeXml; +/** + * Chromedriver is the only binary that does not conform to semantic versioning + * and either has too little number of digits or too many. To get this to be in + * semver, we will either add a '.0' at the end or chop off the last set of + * digits. This is so we can compare to find the latest and greatest. + * + * Example: + * 2.46 -> 2.46.0 + * 75.0.3770.8 -> 75.0.3770 + * + * @param version + */ +function getValidSemver(version) { + let lookUpVersion = ''; + // This supports downloading 2.46 + try { + const oldRegex = /(\d+.\d+)/g; + const exec = oldRegex.exec(version); + if (exec) { + lookUpVersion = exec[1] + '.0'; + } + } + catch (_) { + // no-op: is this is not valid, do not throw here. + } + // This supports downloading 74.0.3729.6 + try { + const newRegex = /(\d+.\d+.\d+).\d+/g; + const exec = newRegex.exec(version); + if (exec) { + lookUpVersion = exec[1]; + } + } + catch (_) { + // no-op: if this does not work, use the other regex pattern. + } + return lookUpVersion; +} +exports.getValidSemver = getValidSemver; +//# sourceMappingURL=chrome_xml.js.map diff --git a/scripts/webdriver-update-newest/update-to-newest-webdriver.sh b/scripts/webdriver-update-newest/update-to-newest-webdriver.sh new file mode 100755 index 00000000000..46990be957e --- /dev/null +++ b/scripts/webdriver-update-newest/update-to-newest-webdriver.sh @@ -0,0 +1,29 @@ +#!/bin/bash +#set -x +BROWSER_TYPE=mac-x64 + +if [ ! -z "$1" ]; then BROWSER_TYPE=$1 ; fi + +PATH_TO_COMMANDS=../../node_modules/webdriver-manager/built/lib/cmds +PATH_TO_BINARIES=../../node_modules/webdriver-manager/built/lib/binaries +PATH_TO_SELENIUM=../../node_modules/webdriver-manager/selenium + +# Remove existing drivers +rm -rf $PATH_TO_SELENIUM/selenium-server-* +rm -rf $PATH_TO_SELENIUM/chromedriver-* +rm -f $PATH_TO_SELENIUM/chromedriver_* + +# Replace browser type in file and create new file +sed "s/mac-x64/$BROWSER_TYPE/" chrome_xml_schema.js > chrome_xml.js +sed "s/mac-x64/$BROWSER_TYPE/" update_schema.js > update.js +sleep 2 + +# Replace webdriver files +cp -f update.js $PATH_TO_COMMANDS/update.js +cp -f chrome_xml.js $PATH_TO_BINARIES/chrome_xml.js + +rm -f update.js +rm -f chrome_xml.js + +#$(npm bin)/webdriver-manager update --gecko=false +node ../../node_modules/webdriver-manager/bin/webdriver-manager update --gecko=false \ No newline at end of file diff --git a/scripts/webdriver-update-newest/update_schema.js b/scripts/webdriver-update-newest/update_schema.js new file mode 100644 index 00000000000..942097d104a --- /dev/null +++ b/scripts/webdriver-update-newest/update_schema.js @@ -0,0 +1,304 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const AdmZip = require("adm-zip"); +const child_process = require("child_process"); +const fs = require("fs"); +const minimist = require("minimist"); +const path = require("path"); +const q = require("q"); +const rimraf = require("rimraf"); +const binaries_1 = require("../binaries"); +const cli_1 = require("../cli"); +const config_1 = require("../config"); +const files_1 = require("../files"); +const http_utils_1 = require("../http_utils"); +const utils_1 = require("../utils"); +const Opt = require("./"); +const initialize_1 = require("./initialize"); +const opts_1 = require("./opts"); +config_1.Config.runCommand = 'update'; +let logger = new cli_1.Logger('update'); +let prog = new cli_1.Program() + .command('update', 'install or update selected binaries') + .action(update) + .addOption(opts_1.Opts[Opt.OUT_DIR]) + .addOption(opts_1.Opts[Opt.VERBOSE]) + .addOption(opts_1.Opts[Opt.IGNORE_SSL]) + .addOption(opts_1.Opts[Opt.PROXY]) + .addOption(opts_1.Opts[Opt.ALTERNATE_CDN]) + .addOption(opts_1.Opts[Opt.STANDALONE]) + .addOption(opts_1.Opts[Opt.CHROME]) + .addOption(opts_1.Opts[Opt.GECKO]) + .addOption(opts_1.Opts[Opt.ANDROID]) + .addOption(opts_1.Opts[Opt.ANDROID_API_LEVELS]) + .addOption(opts_1.Opts[Opt.ANDROID_ARCHITECTURES]) + .addOption(opts_1.Opts[Opt.ANDROID_PLATFORMS]) + .addOption(opts_1.Opts[Opt.ANDROID_ACCEPT_LICENSES]); +if (config_1.Config.osType() === 'Darwin') { + prog.addOption(opts_1.Opts[Opt.IOS]); +} +if (config_1.Config.osType() === 'Windows_NT') { + prog.addOption(opts_1.Opts[Opt.IE]).addOption(opts_1.Opts[Opt.IE32]).addOption(opts_1.Opts[Opt.IE64]); +} +prog.addOption(opts_1.Opts[Opt.VERSIONS_STANDALONE]) + .addOption(opts_1.Opts[Opt.VERSIONS_CHROME]) + .addOption(opts_1.Opts[Opt.VERSIONS_APPIUM]) + .addOption(opts_1.Opts[Opt.VERSIONS_ANDROID]) + .addOption(opts_1.Opts[Opt.VERSIONS_GECKO]); +if (config_1.Config.osType() === 'Windows_NT') { + prog.addOption(opts_1.Opts[Opt.VERSIONS_IE]); +} +exports.program = prog; +// stand alone runner +let argv = minimist(process.argv.slice(2), prog.getMinimistOptions()); +if (argv._[0] === 'update-run') { + prog.run(JSON.parse(JSON.stringify(argv))); +} +else if (argv._[0] === 'update-help') { + prog.printHelp(); +} +let browserFile; +/** + * Parses the options and downloads binaries if they do not exist. + * @param options + */ +function update(options) { + let promises = []; + let standalone = options[Opt.STANDALONE].getBoolean(); + let chrome = options[Opt.CHROME].getBoolean(); + let gecko = options[Opt.GECKO].getBoolean(); + let ie32 = false; + let ie64 = false; + if (options[Opt.IE]) { + ie32 = ie32 || options[Opt.IE].getBoolean(); + } + if (options[Opt.IE32]) { + ie32 = ie32 || options[Opt.IE32].getBoolean(); + } + if (options[Opt.IE64]) { + ie64 = options[Opt.IE64].getBoolean(); + } + let android = options[Opt.ANDROID].getBoolean(); + let ios = false; + if (options[Opt.IOS]) { + ios = options[Opt.IOS].getBoolean(); + } + let outputDir = options[Opt.OUT_DIR].getString(); + try { + browserFile = + JSON.parse(fs.readFileSync(path.resolve(outputDir, 'update-config.json')).toString()); + } + catch (err) { + browserFile = {}; + } + let android_api_levels = options[Opt.ANDROID_API_LEVELS].getString().split(','); + let android_architectures = options[Opt.ANDROID_ARCHITECTURES].getString().split(','); + let android_platforms = options[Opt.ANDROID_PLATFORMS].getString().split(','); + let android_accept_licenses = options[Opt.ANDROID_ACCEPT_LICENSES].getBoolean(); + if (options[Opt.OUT_DIR].getString()) { + if (path.isAbsolute(options[Opt.OUT_DIR].getString())) { + outputDir = options[Opt.OUT_DIR].getString(); + } + else { + outputDir = path.resolve(config_1.Config.getBaseDir(), options[Opt.OUT_DIR].getString()); + } + files_1.FileManager.makeOutputDirectory(outputDir); + } + let ignoreSSL = options[Opt.IGNORE_SSL].getBoolean(); + let proxy = options[Opt.PROXY].getString(); + http_utils_1.HttpUtils.assignOptions({ ignoreSSL, proxy }); + let verbose = options[Opt.VERBOSE].getBoolean(); + // setup versions for binaries + let binaries = files_1.FileManager.setupBinaries(options[Opt.ALTERNATE_CDN].getString()); + binaries[binaries_1.Standalone.id].versionCustom = options[Opt.VERSIONS_STANDALONE].getString(); + binaries[binaries_1.ChromeDriver.id].versionCustom = options[Opt.VERSIONS_CHROME].getString(); + if (options[Opt.VERSIONS_IE]) { + binaries[binaries_1.IEDriver.id].versionCustom = options[Opt.VERSIONS_IE].getString(); + } + if (options[Opt.VERSIONS_GECKO]) { + binaries[binaries_1.GeckoDriver.id].versionCustom = options[Opt.VERSIONS_GECKO].getString(); + } + binaries[binaries_1.AndroidSDK.id].versionCustom = options[Opt.VERSIONS_ANDROID].getString(); + binaries[binaries_1.Appium.id].versionCustom = options[Opt.VERSIONS_APPIUM].getString(); + // if the file has not been completely downloaded, download it + // else if the file has already been downloaded, unzip the file, rename it, and give it + // permissions + if (standalone) { + let binary = binaries[binaries_1.Standalone.id]; + promises.push(files_1.FileManager.downloadFile(binary, outputDir) + .then((downloaded) => { + if (!downloaded) { + logger.info(binary.name + ': file exists ' + + path.resolve(outputDir, binary.filename())); + logger.info(binary.name + ': ' + binary.filename() + ' up to date'); + } + }) + .then(() => { + updateBrowserFile(binary, outputDir); + })); + } + if (chrome) { + let binary = binaries[binaries_1.ChromeDriver.id]; + promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => { + return Promise.resolve(updateBrowserFile(binary, outputDir)); + })); + } + if (gecko) { + let binary = binaries[binaries_1.GeckoDriver.id]; + promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => { + return Promise.resolve(updateBrowserFile(binary, outputDir)); + })); + } + if (ie64) { + let binary = binaries[binaries_1.IEDriver.id]; + binary.osarch = config_1.Config.osArch(); // Win32 or x64 + promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => { + return Promise.resolve(updateBrowserFile(binary, outputDir)); + })); + } + if (ie32) { + let binary = binaries[binaries_1.IEDriver.id]; + binary.osarch = 'Win32'; + promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => { + return Promise.resolve(updateBrowserFile(binary, outputDir)); + })); + } + if (android) { + let binary = binaries[binaries_1.AndroidSDK.id]; + let sdk_path = path.resolve(outputDir, binary.executableFilename()); + let oldAVDList; + updateBrowserFile(binary, outputDir); + promises.push(q.nfcall(fs.readFile, path.resolve(sdk_path, 'available_avds.json')) + .then((oldAVDs) => { + oldAVDList = oldAVDs; + }, () => { + oldAVDList = '[]'; + }) + .then(() => { + return updateBinary(binary, outputDir, proxy, ignoreSSL); + }) + .then(() => { + initialize_1.android(path.resolve(outputDir, binary.executableFilename()), android_api_levels, android_architectures, android_platforms, android_accept_licenses, binaries[binaries_1.AndroidSDK.id].versionCustom, JSON.parse(oldAVDList), logger, verbose); + })); + } + if (ios) { + initialize_1.iOS(logger); + } + if (android || ios) { + installAppium(binaries[binaries_1.Appium.id], outputDir); + updateBrowserFile(binaries[binaries_1.Appium.id], outputDir); + } + return Promise.all(promises).then(() => { + writeBrowserFile(outputDir); + }); +} +function updateBinary(binary, outputDir, proxy, ignoreSSL) { + return files_1.FileManager + .downloadFile(binary, outputDir, (binary, outputDir, fileName) => { + unzip(binary, outputDir, fileName); + }) + .then(downloaded => { + if (!downloaded) { + // The file did not have to download, we should unzip it. + logger.info(binary.name + ': file exists ' + path.resolve(outputDir, binary.filename())); + let fileName = binary.filename(); + unzip(binary, outputDir, fileName); + logger.info(binary.name + ': ' + binary.executableFilename() + ' up to date'); + } + }); +} +function unzip(binary, outputDir, fileName) { + // remove the previously saved file and unzip it + let osType = config_1.Config.osType(); + let mv = path.resolve(outputDir, binary.executableFilename()); + try { + fs.unlinkSync(mv); + } + catch (err) { + try { + rimraf.sync(mv); + } + catch (err2) { + } + } + // unzip the file + logger.info(binary.name + ': unzipping ' + fileName); + if (fileName.slice(-4) == '.zip') { + try { + let zip = new AdmZip(path.resolve(outputDir, fileName)); + zip.extractAllTo(outputDir, true); + } + catch (e) { + throw new Error(`Invalid filename: ${path.resolve(outputDir, fileName)}`); + } + } + else { + // We will only ever get .tar files on linux + child_process.spawnSync('tar', ['zxvf', path.resolve(outputDir, fileName), '-C', outputDir]); + } + // rename + if (fileName.indexOf('chromedriver_') != -1) { + fs.renameSync(path.resolve(outputDir, 'chromedriver-mac-x64', binary.zipContentName()), mv) + } else { + fs.renameSync(path.resolve(outputDir, binary.zipContentName()), mv); + } + // set permissions + if (osType !== 'Windows_NT') { + logger.info(binary.name + ': setting permissions to 0755 for ' + mv); + if (binary.id() !== binaries_1.AndroidSDK.id) { + fs.chmodSync(mv, '0755'); + } + else { + fs.chmodSync(path.resolve(mv, 'tools', 'android'), '0755'); + fs.chmodSync(path.resolve(mv, 'tools', 'emulator'), '0755'); + // TODO(sjelin): get 64 bit versions working + } + } +} +function installAppium(binary, outputDir) { + logger.info('appium: installing appium'); + let folder = path.resolve(outputDir, binary.filename()); + try { + rimraf.sync(folder); + } + catch (err) { + } + fs.mkdirSync(folder); + fs.writeFileSync(path.resolve(folder, 'package.json'), JSON.stringify({ scripts: { appium: 'appium' } })); + utils_1.spawn('npm', ['install', 'appium@' + binary.version()], null, { cwd: folder }); +} +function updateBrowserFile(binary, outputDir) { + let currentDownload = path.resolve(outputDir, binary.executableFilename()); + // if browserFile[id] exists, we should update it + if (browserFile[binary.id()]) { + let binaryPath = browserFile[binary.id()]; + if (binaryPath.last === currentDownload) { + return; + } + else { + binaryPath.last = currentDownload; + for (let bin of binaryPath.all) { + if (bin === currentDownload) { + return; + } + } + binaryPath.all.push(currentDownload); + } + } + else { + // The browserFile[id] does not exist / has not been downloaded previously. + // We should create the entry. + let binaryPath = { last: currentDownload, all: [currentDownload] }; + browserFile[binary.id()] = binaryPath; + } +} +function writeBrowserFile(outputDir) { + let filePath = path.resolve(outputDir, 'update-config.json'); + fs.writeFileSync(filePath, JSON.stringify(browserFile)); +} +// for testing +function clearBrowserFile() { + browserFile = {}; +} +exports.clearBrowserFile = clearBrowserFile; +//# sourceMappingURL=update.js.map \ No newline at end of file