diff --git a/handwritten/storage/conformance-test/libraryMethods.ts b/handwritten/storage/conformance-test/libraryMethods.ts index 4358abe9c1dd..f9836caa1e43 100644 --- a/handwritten/storage/conformance-test/libraryMethods.ts +++ b/handwritten/storage/conformance-test/libraryMethods.ts @@ -797,7 +797,7 @@ export async function createBucket(options: ConformanceTestOptions) { const bucket = options.storage!.bucket('test-creating-bucket'); const [exists] = await bucket.exists(); if (exists) { - bucket.delete(); + await bucket.delete(); } await options.storage!.createBucket('test-creating-bucket'); } diff --git a/handwritten/storage/internal-tooling/performApplicationPerformanceTest.ts b/handwritten/storage/internal-tooling/performApplicationPerformanceTest.ts index eedcda1f6044..68ed32687cfc 100644 --- a/handwritten/storage/internal-tooling/performApplicationPerformanceTest.ts +++ b/handwritten/storage/internal-tooling/performApplicationPerformanceTest.ts @@ -56,7 +56,7 @@ async function main() { ({bucket} = await performanceTestSetup( argv.project! as string, - argv.bucket! as string, + argv.bucket! as string )); switch (argv.test_type) { @@ -78,7 +78,7 @@ async function main() { async function uploadInParallel( bucket: Bucket, paths: string[], - options: UploadOptions, + options: UploadOptions ) { const promises: Promise[] = []; for (const index in paths) { @@ -115,7 +115,7 @@ async function performWriteTest(): Promise { argv.num_objects as number, TEST_NAME_STRING, fileSizeRange.low, - fileSizeRange.high, + fileSizeRange.high ); const start = performance.now(); @@ -157,7 +157,7 @@ async function performReadTest(): Promise { argv.num_objects as number, TEST_NAME_STRING, fileSizeRange.low, - fileSizeRange.high, + fileSizeRange.high ); await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); @@ -188,4 +188,4 @@ async function performReadTest(): Promise { return result; } -main(); +void main(); diff --git a/handwritten/storage/internal-tooling/performPerformanceTest.ts b/handwritten/storage/internal-tooling/performPerformanceTest.ts index 82fda1c9f73a..5a31ace690df 100644 --- a/handwritten/storage/internal-tooling/performPerformanceTest.ts +++ b/handwritten/storage/internal-tooling/performPerformanceTest.ts @@ -54,7 +54,7 @@ async function main() { ({bucket} = await performanceTestSetup( argv.project! as string, - argv.bucket! as string, + argv.bucket! as string )); switch (argv.test_type) { @@ -84,7 +84,7 @@ async function performRangedReadTest(): Promise { fileName, fileSizeRange.low, fileSizeRange.high, - getDirName(), + getDirName() ); const file = bucket.file(`${fileName}`); const destinationFileName = generateRandomFileName(TEST_NAME_STRING); @@ -142,7 +142,7 @@ async function performWriteReadTest(): Promise { fileName, fileSizeRange.low, fileSizeRange.high, - getDirName(), + getDirName() ); for (let j = 0; j < DEFAULT_NUMBER_OF_WRITES; j++) { @@ -214,4 +214,4 @@ async function performWriteReadTest(): Promise { return results; } -main(); +void main(); diff --git a/handwritten/storage/internal-tooling/performTransferManagerTest.ts b/handwritten/storage/internal-tooling/performTransferManagerTest.ts index 6e21e774630d..162d98c66674 100644 --- a/handwritten/storage/internal-tooling/performTransferManagerTest.ts +++ b/handwritten/storage/internal-tooling/performTransferManagerTest.ts @@ -56,7 +56,7 @@ async function main() { ({bucket, transferManager} = await performanceTestSetup( argv.project! as string, - argv.bucket! as string, + argv.bucket! as string )); switch (argv.test_type) { @@ -83,11 +83,8 @@ async function performTestCleanup(fileOrFiles: File[] | File | string[]) { ? fileOrFiles : [fileOrFiles]; const promises = filesToDelete.map(f => { - let fileToDelete = f; - if (typeof f === 'string') { - fileToDelete = bucket.file(f); - } - (fileToDelete as File).delete({ignoreNotFound: true}); + const fileToDelete = typeof f === 'string' ? bucket.file(f) : f; + return fileToDelete.delete({ignoreNotFound: true}); }); return Promise.all(promises); } @@ -104,7 +101,7 @@ async function performUploadManyFilesTest(): Promise { TEST_NAME_STRING, fileSizeRange.low, fileSizeRange.high, - DIRECTORY_PROBABILITY, + DIRECTORY_PROBABILITY ); const start = performance.now(); @@ -153,7 +150,7 @@ async function performDownloadManyFilesTest(): Promise { TEST_NAME_STRING, fileSizeRange.low, fileSizeRange.high, - DIRECTORY_PROBABILITY, + DIRECTORY_PROBABILITY ); await transferManager.uploadManyFiles(creationInfo.paths, { @@ -210,7 +207,7 @@ async function performChunkUploadDownloadTest(): Promise { fileName, fileSizeRange.low, fileSizeRange.high, - getDirName(), + getDirName() ); const file = bucket.file(`${fileName}`); let result: TestResult = { @@ -276,4 +273,4 @@ async function performChunkUploadDownloadTest(): Promise { return results; } -main(); +void main(); diff --git a/handwritten/storage/internal-tooling/performanceTest.ts b/handwritten/storage/internal-tooling/performanceTest.ts index 02adc10f76fa..33e6a90cfa3a 100644 --- a/handwritten/storage/internal-tooling/performanceTest.ts +++ b/handwritten/storage/internal-tooling/performanceTest.ts @@ -44,7 +44,7 @@ function main() { if (numThreads > iterationsRemaining) { log( `${numThreads} is greater than number of iterations (${iterationsRemaining}). Using ${iterationsRemaining} threads instead.`, - argv.debug as boolean, + argv.debug as boolean ); numThreads = iterationsRemaining; } @@ -65,7 +65,7 @@ function createWorker() { iterationsRemaining--; log( `Starting new iteration. Current iterations remaining: ${iterationsRemaining}`, - argv.debug as boolean, + argv.debug as boolean ); let testPath = ''; if ( @@ -98,7 +98,7 @@ function createWorker() { w.on('message', data => { log('Successfully completed iteration.', argv.debug as boolean); - recordResult(data); + recordResult(data).catch(err => log(err, argv.debug as boolean, true)); if (iterationsRemaining > 0) { createWorker(); } @@ -122,7 +122,7 @@ async function recordResult(results: TestResult[] | TestResult) { : [results]; for await (const outputString of convertToCloudMonitoringFormat( - resultsToAppend, + resultsToAppend )) { argv.file_name ? await appendFile(argv.file_name as string, `${outputString}\n`) diff --git a/handwritten/storage/internal-tooling/performanceUtils.ts b/handwritten/storage/internal-tooling/performanceUtils.ts index b0633bdf1c4d..8d1d0aed898e 100644 --- a/handwritten/storage/internal-tooling/performanceUtils.ts +++ b/handwritten/storage/internal-tooling/performanceUtils.ts @@ -14,12 +14,13 @@ * limitations under the License. */ -import {execSync} from 'child_process'; import {mkdirSync, mkdtempSync, unlinkSync} from 'fs'; import * as path from 'path'; import * as yargs from 'yargs'; import {Bucket, Storage, TransferManager} from '../src/index.js'; import {getDirName} from '../src/util.js'; +import * as crypto from 'crypto'; +import * as fs from 'fs'; export const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; export const DEFAULT_DIRECTORY_PROBABILITY = 0.1; @@ -156,7 +157,7 @@ export interface PerformanceTestSetupResults { } /** - * Create a uniformly distributed random integer beween the inclusive min and max provided. + * Create a uniformly distributed random integer between the inclusive min and max provided. * * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return. * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return. @@ -172,12 +173,12 @@ export function randomInteger(minInclusive: number, maxInclusive: number) { /** * Returns a boolean value with the provided probability * - * @param {number} trueProbablity the probability the value will be true + * @param {number} trueProbability the probability the value will be true * - * @returns {boolean} a boolean value with the probablity provided. + * @returns {boolean} a boolean value with the probability provided. */ -export function weightedRandomBoolean(trueProbablity: number): boolean { - return Math.random() <= trueProbablity ? true : false; +export function weightedRandomBoolean(trueProbability: number): boolean { + return Math.random() <= trueProbability ? true : false; } /** @@ -214,21 +215,21 @@ export function generateRandomFileName(baseName: string): string { export function generateRandomFile( fileName: string, fileSizeLowerBoundBytes: number = getLowHighFileSize( - DEFAULT_OBJECT_RANGE_SIZE_BYTES, + DEFAULT_OBJECT_RANGE_SIZE_BYTES ).low, fileSizeUpperBoundBytes: number = getLowHighFileSize( - DEFAULT_OBJECT_RANGE_SIZE_BYTES, + DEFAULT_OBJECT_RANGE_SIZE_BYTES ).high, - currentDirectory: string = mkdtempSync(randomString()), + currentDirectory: string = mkdtempSync(randomString()) ): number { const fileSizeBytes = randomInteger( fileSizeLowerBoundBytes, - fileSizeUpperBoundBytes, + fileSizeUpperBoundBytes ); - execSync( - `head --bytes=${fileSizeBytes} /dev/urandom > ${currentDirectory}/${fileName}`, - ); + const filePath = path.join(currentDirectory, fileName); + const randomBuffer = crypto.randomBytes(fileSizeBytes); + fs.writeFileSync(filePath, randomBuffer); return fileSizeBytes; } @@ -247,12 +248,12 @@ export function generateRandomDirectoryStructure( maxObjects: number, baseName: string, fileSizeLowerBoundBytes: number = getLowHighFileSize( - DEFAULT_OBJECT_RANGE_SIZE_BYTES, + DEFAULT_OBJECT_RANGE_SIZE_BYTES ).low, fileSizeUpperBoundBytes: number = getLowHighFileSize( - DEFAULT_OBJECT_RANGE_SIZE_BYTES, + DEFAULT_OBJECT_RANGE_SIZE_BYTES ).high, - directoryProbability: number = DEFAULT_DIRECTORY_PROBABILITY, + directoryProbability: number = DEFAULT_DIRECTORY_PROBABILITY ): RandomDirectoryCreationInformation { let curPath = baseName; const creationInfo: RandomDirectoryCreationInformation = { @@ -272,7 +273,7 @@ export function generateRandomDirectoryStructure( randomName, fileSizeLowerBoundBytes, fileSizeUpperBoundBytes, - curPath, + curPath ); creationInfo.paths.push(path.join(curPath, randomName)); } @@ -288,7 +289,7 @@ export function generateRandomDirectoryStructure( */ export function cleanupFile( fileName: string, - directoryName: string = getDirName(), + directoryName: string = getDirName() ): void { unlinkSync(`${directoryName}/${fileName}`); } @@ -302,7 +303,7 @@ export function cleanupFile( */ export async function performanceTestSetup( projectId: string, - bucketName: string, + bucketName: string ): Promise { const storage = new Storage({projectId}); const bucket = storage.bucket(bucketName, { @@ -346,7 +347,7 @@ export function getValidationType(): 'md5' | 'crc32c' | boolean | undefined { * @returns {AsyncGenerator} A string containing the results of the conversion to cloud monitoring format. */ export async function* convertToCloudMonitoringFormat( - results: TestResult[], + results: TestResult[] ): AsyncGenerator { for (const curResult of results) { const throughput = @@ -384,7 +385,7 @@ export async function* convertToCloudMonitoringFormat( export function log( messageOrError: string | Error, shouldLog: boolean, - isError = false, + isError = false ): void { if (shouldLog) { isError ? console.error(messageOrError) : console.log(messageOrError);