From 248e3651e341146f6ffa442ef56745b37a617e2b Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 14 Dec 2021 12:22:48 +0000 Subject: [PATCH 01/45] Check input combinations --- dist/index.js | 16 +++++++++++----- src/input.ts | 18 ++++++++++++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/dist/index.js b/dist/index.js index 13c7f30..c7a1f14 100644 --- a/dist/index.js +++ b/dist/index.js @@ -71,12 +71,13 @@ exports.deleteVersions = deleteVersions; /***/ }), /***/ 8657: -/***/ ((__unused_webpack_module, exports) => { +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); exports.Input = void 0; +const rxjs_1 = __nccwpck_require__(5805); const defaultParams = { packageVersionIds: [], owner: '', @@ -100,13 +101,18 @@ class Input { this.ignoreVersions = validatedParams.ignoreVersions; this.deletePreReleaseVersions = validatedParams.deletePreReleaseVersions; this.token = validatedParams.token; - if (this.minVersionsToKeep > 0) { - this.numOldVersionsToDelete = 100 - this.minVersionsToKeep; + if (this.numOldVersionsToDelete > 1 && + (this.minVersionsToKeep >= 0 || this.deletePreReleaseVersions == 'true')) { + rxjs_1.throwError('Invalid input combination'); } - if (this.deletePreReleaseVersions == 'true') { - this.numOldVersionsToDelete = 100 - this.minVersionsToKeep; + if (this.deletePreReleaseVersions === 'true') { + this.minVersionsToKeep = + this.minVersionsToKeep > 0 ? this.minVersionsToKeep : 0; this.ignoreVersions = new RegExp('^(0|[1-9]\\d*)((\\.(0|[1-9]\\d*))*)$'); } + if (this.minVersionsToKeep >= 0) { + this.numOldVersionsToDelete = 0; + } } hasOldestVersionQueryInfo() { return !!(this.owner && diff --git a/src/input.ts b/src/input.ts index 8685fd6..0d8a9c6 100644 --- a/src/input.ts +++ b/src/input.ts @@ -1,3 +1,5 @@ +import {throwError} from 'rxjs' + export interface InputParams { packageVersionIds?: string[] owner?: string @@ -46,14 +48,22 @@ export class Input { this.deletePreReleaseVersions = validatedParams.deletePreReleaseVersions this.token = validatedParams.token - if (this.minVersionsToKeep > 0) { - this.numOldVersionsToDelete = 100 - this.minVersionsToKeep + if ( + this.numOldVersionsToDelete > 1 && + (this.minVersionsToKeep >= 0 || this.deletePreReleaseVersions == 'true') + ) { + throwError('Invalid input combination') } - if (this.deletePreReleaseVersions == 'true') { - this.numOldVersionsToDelete = 100 - this.minVersionsToKeep + if (this.deletePreReleaseVersions === 'true') { + this.minVersionsToKeep = + this.minVersionsToKeep > 0 ? this.minVersionsToKeep : 0 this.ignoreVersions = new RegExp('^(0|[1-9]\\d*)((\\.(0|[1-9]\\d*))*)$') } + + if (this.minVersionsToKeep >= 0) { + this.numOldVersionsToDelete = 0 + } } hasOldestVersionQueryInfo(): boolean { From c253b9d27aa526d314c90a543ce8223a847de235 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 14 Dec 2021 12:34:03 +0000 Subject: [PATCH 02/45] Check input combinations --- dist/index.js | 29 +++++++++++++++++------------ src/delete.ts | 4 ++++ src/input.ts | 28 ++++++++++++++++------------ 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/dist/index.js b/dist/index.js index c7a1f14..c3567ce 100644 --- a/dist/index.js +++ b/dist/index.js @@ -59,6 +59,9 @@ function deleteVersions(input) { if (!input.token) { return rxjs_1.throwError('No token found'); } + if (!input.checkInput()) { + return rxjs_1.throwError('Invalid input combination'); + } if (input.numOldVersionsToDelete <= 0) { console.log('Number of old versions to delete input is 0 or less, no versions will be deleted'); return rxjs_1.of(true); @@ -71,13 +74,12 @@ exports.deleteVersions = deleteVersions; /***/ }), /***/ 8657: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +/***/ ((__unused_webpack_module, exports) => { "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); exports.Input = void 0; -const rxjs_1 = __nccwpck_require__(5805); const defaultParams = { packageVersionIds: [], owner: '', @@ -101,9 +103,19 @@ class Input { this.ignoreVersions = validatedParams.ignoreVersions; this.deletePreReleaseVersions = validatedParams.deletePreReleaseVersions; this.token = validatedParams.token; + } + hasOldestVersionQueryInfo() { + return !!(this.owner && + this.repo && + this.packageName && + this.numOldVersionsToDelete > 0 && + this.minVersionsToKeep >= 0 && + this.token); + } + checkInput() { if (this.numOldVersionsToDelete > 1 && - (this.minVersionsToKeep >= 0 || this.deletePreReleaseVersions == 'true')) { - rxjs_1.throwError('Invalid input combination'); + (this.minVersionsToKeep >= 0 || this.deletePreReleaseVersions === 'true')) { + return false; } if (this.deletePreReleaseVersions === 'true') { this.minVersionsToKeep = @@ -113,14 +125,7 @@ class Input { if (this.minVersionsToKeep >= 0) { this.numOldVersionsToDelete = 0; } - } - hasOldestVersionQueryInfo() { - return !!(this.owner && - this.repo && - this.packageName && - this.numOldVersionsToDelete > 0 && - this.minVersionsToKeep >= 0 && - this.token); + return true; } } exports.Input = Input; diff --git a/src/delete.ts b/src/delete.ts index 0b9faa7..595ce1d 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -49,6 +49,10 @@ export function deleteVersions(input: Input): Observable { return throwError('No token found') } + if (!input.checkInput()) { + return throwError('Invalid input combination') + } + if (input.numOldVersionsToDelete <= 0) { console.log( 'Number of old versions to delete input is 0 or less, no versions will be deleted' diff --git a/src/input.ts b/src/input.ts index 0d8a9c6..c016c66 100644 --- a/src/input.ts +++ b/src/input.ts @@ -47,12 +47,25 @@ export class Input { this.ignoreVersions = validatedParams.ignoreVersions this.deletePreReleaseVersions = validatedParams.deletePreReleaseVersions this.token = validatedParams.token + } + + hasOldestVersionQueryInfo(): boolean { + return !!( + this.owner && + this.repo && + this.packageName && + this.numOldVersionsToDelete > 0 && + this.minVersionsToKeep >= 0 && + this.token + ) + } + checkInput(): boolean { if ( this.numOldVersionsToDelete > 1 && - (this.minVersionsToKeep >= 0 || this.deletePreReleaseVersions == 'true') + (this.minVersionsToKeep >= 0 || this.deletePreReleaseVersions === 'true') ) { - throwError('Invalid input combination') + return false } if (this.deletePreReleaseVersions === 'true') { @@ -64,16 +77,7 @@ export class Input { if (this.minVersionsToKeep >= 0) { this.numOldVersionsToDelete = 0 } - } - hasOldestVersionQueryInfo(): boolean { - return !!( - this.owner && - this.repo && - this.packageName && - this.numOldVersionsToDelete > 0 && - this.minVersionsToKeep >= 0 && - this.token - ) + return true } } From e7ecd94635d092f0fc51d6b49322c2bad0d59719 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 14 Dec 2021 12:39:22 +0000 Subject: [PATCH 03/45] change default values --- action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index 7f21c6f..9a46391 100644 --- a/action.yml +++ b/action.yml @@ -37,9 +37,10 @@ inputs: min-versions-to-keep: description: > Number of versions to keep starting with the latest version - Defaults to 0. + By default keeps no version. + To delete all versions set this as 0. required: false - default: "0" + default: "-1" ignore-versions: description: > @@ -50,7 +51,7 @@ inputs: delete-only-pre-release-versions: description: > - Deletes only pre-release versions upto. The number of pre-release versions to keep can be specified by min-versions-to-keep. + Deletes only pre-release versions. The number of pre-release versions to keep can be specified by min-versions-to-keep. When this is set num-old-versions-to-delete and ignore-versions will not be taken into account. By default this is set to false required: false From 93d3863084c9505679df0e6eae4ca82b4fe2a6a4 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 14 Dec 2021 15:51:31 +0000 Subject: [PATCH 04/45] add paging --- dist/index.js | 185 +++++++++++++++++++++++++++--------- package.json | 2 +- src/delete.ts | 163 +++++++++++++++++++++++-------- src/version/get-versions.ts | 137 +++++++++++++++++++++----- 4 files changed, 377 insertions(+), 110 deletions(-) diff --git a/dist/index.js b/dist/index.js index c3567ce..11fa4da 100644 --- a/dist/index.js +++ b/dist/index.js @@ -23,50 +23,79 @@ module.exports = JSON.parse('{"_args":[["@octokit/rest@16.43.1","/workspaces/del "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.deleteVersions = exports.getVersionIds = void 0; +exports.deleteVersions = exports.finalIds = exports.getVersionIds = void 0; const rxjs_1 = __nccwpck_require__(5805); const version_1 = __nccwpck_require__(4428); const operators_1 = __nccwpck_require__(7801); -function getVersionIds(input) { +let totalCount; +function getVersionIds(owner, repo, packageName, numVersions, ignoreVersions, cursor, token) { + return version_1.getOldestVersions(owner, repo, packageName, numVersions, ignoreVersions, cursor, token).pipe(operators_1.expand(value => value.paginate + ? version_1.getOldestVersions(owner, repo, packageName, numVersions, ignoreVersions, value.cursor, token) + : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.map(value => value.versions), operators_1.tap(value => value.map(info => console.log(`id0: ${info.id}, version: ${info.version}`)))); +} +exports.getVersionIds = getVersionIds; +function finalIds(input) { if (input.packageVersionIds.length > 0) { return rxjs_1.of(input.packageVersionIds); } if (input.hasOldestVersionQueryInfo()) { - return version_1.getOldestVersions(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete + input.minVersionsToKeep, input.token).pipe(operators_1.map(versionInfo => { - const numberVersionsToDelete = versionInfo.length - input.minVersionsToKeep; - if (input.minVersionsToKeep > 0) { - return numberVersionsToDelete <= 0 - ? [] - : versionInfo - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, -input.minVersionsToKeep); - } - else { - return numberVersionsToDelete <= 0 - ? [] - : versionInfo - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, numberVersionsToDelete); - } - })); + if (input.minVersionsToKeep < 0) { + console.log(`in numOldVersionsToDelete`); + return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, input.ignoreVersions, '', input.token).pipe(operators_1.map(value => { + const temp = input.numOldVersionsToDelete; + input.numOldVersionsToDelete = + input.numOldVersionsToDelete - value.length <= 0 + ? 0 + : input.numOldVersionsToDelete - value.length; + console.log(`temp: ${temp} numVersions: ${input.numOldVersionsToDelete} ignore-versions: ${input.ignoreVersions}`); + return value + .filter(info => !input.ignoreVersions.test(info.version)) + .map(info => info.id) + .slice(0, temp); + })); + } + else { + console.log(`in min versions to keep`); + return getVersionIds(input.owner, input.repo, input.packageName, 100, input.ignoreVersions, '', input.token).pipe(operators_1.map(value => { + console.log(`point 1`); + const toDelete = totalCount - + value.filter(info => !input.ignoreVersions.test(info.version)) + .length - + input.minVersionsToKeep; + console.log(`toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}`); + if (toDelete > input.numOldVersionsToDelete) { + //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions + input.numOldVersionsToDelete = + input.numOldVersionsToDelete + value.length; + return toDelete - input.numOldVersionsToDelete >= 0 + ? value + .filter(info => !input.ignoreVersions.test(info.version)) + .map(info => info.id) + : value + .filter(info => !input.ignoreVersions.test(info.version)) + .map(info => info.id) + .slice(0, toDelete - input.numOldVersionsToDelete); + } + else + return []; + })); + } } - return rxjs_1.throwError("Could not get packageVersionIds. Explicitly specify using the 'package-version-ids' input or provide the 'package-name' and 'num-old-versions-to-delete' inputs to dynamically retrieve oldest versions"); + return rxjs_1.throwError(`no package id found`); } -exports.getVersionIds = getVersionIds; +exports.finalIds = finalIds; function deleteVersions(input) { if (!input.token) { return rxjs_1.throwError('No token found'); } if (!input.checkInput()) { - return rxjs_1.throwError('Invalid input combination'); + return rxjs_1.throwError('Invlaid input combination'); } - if (input.numOldVersionsToDelete <= 0) { + if (input.numOldVersionsToDelete <= 0 && input.minVersionsToKeep < 0) { console.log('Number of old versions to delete input is 0 or less, no versions will be deleted'); return rxjs_1.of(true); } - return getVersionIds(input).pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); + return finalIds(input).pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); } exports.deleteVersions = deleteVersions; @@ -201,48 +230,112 @@ const query = ` node { name versions(last: $last) { + totalCount edges { node { id version } } + pageInfo { + startCursor + hasPreviousPage + } } } } } } }`; -function queryForOldestVersions(owner, repo, packageName, numVersions, token) { - return rxjs_1.from(graphql_1.graphql(token, query, { - owner, - repo, - package: packageName, - last: numVersions, - headers: { - Accept: 'application/vnd.github.packages-preview+json' +const Paginatequery = ` + query getVersions($owner: String!, $repo: String!, $package: String!, $last: Int!, $before: String!) { + repository(owner: $owner, name: $repo) { + packages(first: 1, names: [$package]) { + edges { + node { + name + versions(last: $last, before: $before) { + totalCount + edges { + node { + id + version + } + } + pageInfo{ + startCursor + hasPreviousPage + } + } + } } - })).pipe(operators_1.catchError((err) => { - const msg = 'query for oldest version failed.'; - return rxjs_1.throwError(err.errors && err.errors.length > 0 - ? `${msg} ${err.errors[0].message}` - : `${msg} verify input parameters are correct`); - })); + } + } + }`; +function queryForOldestVersions(owner, repo, packageName, numVersions, startCursor, token) { + if (startCursor === '') { + return rxjs_1.from(graphql_1.graphql(token, query, { + owner, + repo, + package: packageName, + last: numVersions > 100 ? 100 : numVersions, + headers: { + Accept: 'application/vnd.github.packages-preview+json' + } + })).pipe(operators_1.catchError((err) => { + const msg = 'query for oldest version failed.'; + return rxjs_1.throwError(err.errors && err.errors.length > 0 + ? `${msg} ${err.errors[0].message}` + : `${msg} verify input parameters are correct`); + })); + } + else { + return rxjs_1.from(graphql_1.graphql(token, Paginatequery, { + owner, + repo, + package: packageName, + last: numVersions > 100 ? 100 : numVersions, + before: startCursor, + headers: { + Accept: 'application/vnd.github.packages-preview+json' + } + })).pipe(operators_1.catchError((err) => { + const msg = 'query for oldest version failed.'; + return rxjs_1.throwError(err.errors && err.errors.length > 0 + ? `${msg} ${err.errors[0].message}` + : `${msg} verify input parameters are correct`); + })); + } } exports.queryForOldestVersions = queryForOldestVersions; -function getOldestVersions(owner, repo, packageName, numVersions, token) { - return queryForOldestVersions(owner, repo, packageName, numVersions, token).pipe(operators_1.map(result => { +function getOldestVersions(owner, repo, packageName, numVersions, ignoreVersions, startCursor, token) { + return queryForOldestVersions(owner, repo, packageName, numVersions, startCursor, token).pipe(operators_1.map(result => { + let r; if (result.repository.packages.edges.length < 1) { console.log(`package: ${packageName} not found for owner: ${owner} in repo: ${repo}`); - return []; + r = { + versions: [], + cursor: '', + paginate: false, + totalCount: 0 + }; + return r; } const versions = result.repository.packages.edges[0].node.versions.edges; + const pages = result.repository.packages.edges[0].node.versions.pageInfo; + const count = result.repository.packages.edges[0].node.versions.totalCount; if (versions.length !== numVersions) { console.log(`number of versions requested was: ${numVersions}, but found: ${versions.length}`); } - return versions - .map(value => ({ id: value.node.id, version: value.node.version })) - .reverse(); + r = { + versions: versions + .map(value => ({ id: value.node.id, version: value.node.version })) + .reverse(), + cursor: pages.startCursor, + paginate: pages.hasPreviousPage, + totalCount: count + }; + return r; })); } exports.getOldestVersions = getOldestVersions; diff --git a/package.json b/package.json index 20cdae1..ca8b82d 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "format-check": "prettier --check **/*.ts", "lint": "eslint src/**/*.ts", "test": "jest", - "build": "npm run format-check && npm run lint && npm run test && tsc", + "build": "npm run format-check && npm run lint && tsc", "pack": "rm -rf ./lib ./dist && npm run build && ncc build" }, "repository": { diff --git a/src/delete.ts b/src/delete.ts index 595ce1d..fcb875d 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -1,47 +1,134 @@ import {Input} from './input' -import {Observable, of, throwError} from 'rxjs' +import {EMPTY, Observable, of, throwError} from 'rxjs' import {deletePackageVersions, getOldestVersions} from './version' -import {concatMap, map} from 'rxjs/operators' +import {concatMap, map, expand, tap} from 'rxjs/operators' -export function getVersionIds(input: Input): Observable { +export interface VersionInfo { + id: string + version: string +} + +export interface QueryInfo { + versions: VersionInfo[] + cursor: string + paginate: boolean + totalCount: number +} + +let totalCount: number + +export function getVersionIds( + owner: string, + repo: string, + packageName: string, + numVersions: number, + ignoreVersions: RegExp, + cursor: string, + token: string +): Observable { + return getOldestVersions( + owner, + repo, + packageName, + numVersions, + ignoreVersions, + cursor, + token + ).pipe( + expand(value => + value.paginate + ? getOldestVersions( + owner, + repo, + packageName, + numVersions, + ignoreVersions, + value.cursor, + token + ) + : EMPTY + ), + tap(value => (totalCount = value.totalCount)), + map(value => value.versions), + tap(value => + value.map(info => + console.log(`id0: ${info.id}, version: ${info.version}`) + ) + ) + ) +} + +export function finalIds(input: Input): Observable { if (input.packageVersionIds.length > 0) { return of(input.packageVersionIds) } - if (input.hasOldestVersionQueryInfo()) { - return getOldestVersions( - input.owner, - input.repo, - input.packageName, - input.numOldVersionsToDelete + input.minVersionsToKeep, - input.token - ).pipe( - map(versionInfo => { - const numberVersionsToDelete = - versionInfo.length - input.minVersionsToKeep - - if (input.minVersionsToKeep > 0) { - return numberVersionsToDelete <= 0 - ? [] - : versionInfo - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, -input.minVersionsToKeep) - } else { - return numberVersionsToDelete <= 0 - ? [] - : versionInfo - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, numberVersionsToDelete) - } - }) - ) + if (input.minVersionsToKeep < 0) { + console.log(`in numOldVersionsToDelete`) + return getVersionIds( + input.owner, + input.repo, + input.packageName, + input.numOldVersionsToDelete, + input.ignoreVersions, + '', + input.token + ).pipe( + map(value => { + const temp = input.numOldVersionsToDelete + input.numOldVersionsToDelete = + input.numOldVersionsToDelete - value.length <= 0 + ? 0 + : input.numOldVersionsToDelete - value.length + console.log( + `temp: ${temp} numVersions: ${input.numOldVersionsToDelete} ignore-versions: ${input.ignoreVersions}` + ) + return value + .filter(info => !input.ignoreVersions.test(info.version)) + .map(info => info.id) + .slice(0, temp) + }) + ) + } else { + console.log(`in min versions to keep`) + return getVersionIds( + input.owner, + input.repo, + input.packageName, + 100, + input.ignoreVersions, + '', + input.token + ).pipe( + map(value => { + console.log(`point 1`) + const toDelete = + totalCount - + value.filter(info => !input.ignoreVersions.test(info.version)) + .length - + input.minVersionsToKeep + console.log( + `toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}` + ) + if (toDelete > input.numOldVersionsToDelete) { + //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions + input.numOldVersionsToDelete = + input.numOldVersionsToDelete + value.length + return toDelete - input.numOldVersionsToDelete >= 0 + ? value + .filter(info => !input.ignoreVersions.test(info.version)) + .map(info => info.id) + : value + .filter(info => !input.ignoreVersions.test(info.version)) + .map(info => info.id) + .slice(0, toDelete - input.numOldVersionsToDelete) + } else return [] + }) + ) + } } - return throwError( - "Could not get packageVersionIds. Explicitly specify using the 'package-version-ids' input or provide the 'package-name' and 'num-old-versions-to-delete' inputs to dynamically retrieve oldest versions" - ) + return throwError(`no package id found`) } export function deleteVersions(input: Input): Observable { @@ -50,17 +137,17 @@ export function deleteVersions(input: Input): Observable { } if (!input.checkInput()) { - return throwError('Invalid input combination') + return throwError('Invlaid input combination') } - if (input.numOldVersionsToDelete <= 0) { + if (input.numOldVersionsToDelete <= 0 && input.minVersionsToKeep < 0) { console.log( 'Number of old versions to delete input is 0 or less, no versions will be deleted' ) return of(true) } - return getVersionIds(input).pipe( + return finalIds(input).pipe( concatMap(ids => deletePackageVersions(ids, input.token)) ) } diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index 8e97f60..099556f 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -8,6 +8,13 @@ export interface VersionInfo { version: string } +export interface QueryInfo { + versions: VersionInfo[] + cursor: string + paginate: boolean + totalCount: number +} + export interface GetVersionsQueryResponse { repository: { packages: { @@ -15,7 +22,12 @@ export interface GetVersionsQueryResponse { node: { name: string versions: { + totalCount: number edges: {node: VersionInfo}[] + pageInfo: { + startCursor: string + hasPreviousPage: boolean + } } } }[] @@ -31,12 +43,43 @@ const query = ` node { name versions(last: $last) { + totalCount edges { node { id version } } + pageInfo { + startCursor + hasPreviousPage + } + } + } + } + } + } + }` + +const Paginatequery = ` + query getVersions($owner: String!, $repo: String!, $package: String!, $last: Int!, $before: String!) { + repository(owner: $owner, name: $repo) { + packages(first: 1, names: [$package]) { + edges { + node { + name + versions(last: $last, before: $before) { + totalCount + edges { + node { + id + version + } + } + pageInfo{ + startCursor + hasPreviousPage + } } } } @@ -49,28 +92,53 @@ export function queryForOldestVersions( repo: string, packageName: string, numVersions: number, + startCursor: string, token: string ): Observable { - return from( - graphql(token, query, { - owner, - repo, - package: packageName, - last: numVersions, - headers: { - Accept: 'application/vnd.github.packages-preview+json' - } - }) as Promise - ).pipe( - catchError((err: GraphQlQueryResponse) => { - const msg = 'query for oldest version failed.' - return throwError( - err.errors && err.errors.length > 0 - ? `${msg} ${err.errors[0].message}` - : `${msg} verify input parameters are correct` - ) - }) - ) + if (startCursor === '') { + return from( + graphql(token, query, { + owner, + repo, + package: packageName, + last: numVersions > 100 ? 100 : numVersions, + headers: { + Accept: 'application/vnd.github.packages-preview+json' + } + }) as Promise + ).pipe( + catchError((err: GraphQlQueryResponse) => { + const msg = 'query for oldest version failed.' + return throwError( + err.errors && err.errors.length > 0 + ? `${msg} ${err.errors[0].message}` + : `${msg} verify input parameters are correct` + ) + }) + ) + } else { + return from( + graphql(token, Paginatequery, { + owner, + repo, + package: packageName, + last: numVersions > 100 ? 100 : numVersions, + before: startCursor, + headers: { + Accept: 'application/vnd.github.packages-preview+json' + } + }) as Promise + ).pipe( + catchError((err: GraphQlQueryResponse) => { + const msg = 'query for oldest version failed.' + return throwError( + err.errors && err.errors.length > 0 + ? `${msg} ${err.errors[0].message}` + : `${msg} verify input parameters are correct` + ) + }) + ) + } } export function getOldestVersions( @@ -78,24 +146,36 @@ export function getOldestVersions( repo: string, packageName: string, numVersions: number, + ignoreVersions: RegExp, + startCursor: string, token: string -): Observable { +): Observable { return queryForOldestVersions( owner, repo, packageName, numVersions, + startCursor, token ).pipe( map(result => { + let r: QueryInfo if (result.repository.packages.edges.length < 1) { console.log( `package: ${packageName} not found for owner: ${owner} in repo: ${repo}` ) - return [] + r = { + versions: [], + cursor: '', + paginate: false, + totalCount: 0 + } + return r } const versions = result.repository.packages.edges[0].node.versions.edges + const pages = result.repository.packages.edges[0].node.versions.pageInfo + const count = result.repository.packages.edges[0].node.versions.totalCount if (versions.length !== numVersions) { console.log( @@ -103,9 +183,16 @@ export function getOldestVersions( ) } - return versions - .map(value => ({id: value.node.id, version: value.node.version})) - .reverse() + r = { + versions: versions + .map(value => ({id: value.node.id, version: value.node.version})) + .reverse(), + cursor: pages.startCursor, + paginate: pages.hasPreviousPage, + totalCount: count + } + + return r }) ) } From 334f20f7fa4a08e80d182d9264a98a5e610d6bce Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 14 Dec 2021 15:57:21 +0000 Subject: [PATCH 05/45] fix checks --- dist/index.js | 5 ++--- src/delete.ts | 4 +++- src/input.ts | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dist/index.js b/dist/index.js index 11fa4da..94c12ce 100644 --- a/dist/index.js +++ b/dist/index.js @@ -81,7 +81,7 @@ function finalIds(input) { })); } } - return rxjs_1.throwError(`no package id found`); + return rxjs_1.throwError("Could not get packageVersionIds. Explicitly specify using the 'package-version-ids' input"); } exports.finalIds = finalIds; function deleteVersions(input) { @@ -137,8 +137,7 @@ class Input { return !!(this.owner && this.repo && this.packageName && - this.numOldVersionsToDelete > 0 && - this.minVersionsToKeep >= 0 && + this.numOldVersionsToDelete >= 0 && this.token); } checkInput() { diff --git a/src/delete.ts b/src/delete.ts index fcb875d..91ce8de 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -128,7 +128,9 @@ export function finalIds(input: Input): Observable { } } - return throwError(`no package id found`) + return throwError( + "Could not get packageVersionIds. Explicitly specify using the 'package-version-ids' input" + ) } export function deleteVersions(input: Input): Observable { diff --git a/src/input.ts b/src/input.ts index c016c66..dd996ca 100644 --- a/src/input.ts +++ b/src/input.ts @@ -54,8 +54,7 @@ export class Input { this.owner && this.repo && this.packageName && - this.numOldVersionsToDelete > 0 && - this.minVersionsToKeep >= 0 && + this.numOldVersionsToDelete >= 0 && this.token ) } From 0994e6c0d2243bfdbb673f790fda6e6508e3c2d0 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 14 Dec 2021 16:06:05 +0000 Subject: [PATCH 06/45] fix delete count update condition --- dist/index.js | 2 +- src/delete.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index 94c12ce..99233ee 100644 --- a/dist/index.js +++ b/dist/index.js @@ -59,7 +59,7 @@ function finalIds(input) { return getVersionIds(input.owner, input.repo, input.packageName, 100, input.ignoreVersions, '', input.token).pipe(operators_1.map(value => { console.log(`point 1`); const toDelete = totalCount - - value.filter(info => !input.ignoreVersions.test(info.version)) + value.filter(info => input.ignoreVersions.test(info.version)) .length - input.minVersionsToKeep; console.log(`toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}`); diff --git a/src/delete.ts b/src/delete.ts index 91ce8de..8684c9f 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -104,7 +104,7 @@ export function finalIds(input: Input): Observable { console.log(`point 1`) const toDelete = totalCount - - value.filter(info => !input.ignoreVersions.test(info.version)) + value.filter(info => input.ignoreVersions.test(info.version)) .length - input.minVersionsToKeep console.log( From 8c24449b5635f821c7b8b36b863cb2b8a66a37d9 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 14 Dec 2021 16:15:58 +0000 Subject: [PATCH 07/45] fix delete all pre release versions --- dist/index.js | 6 ++---- src/delete.ts | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/dist/index.js b/dist/index.js index 99233ee..7905c9c 100644 --- a/dist/index.js +++ b/dist/index.js @@ -62,17 +62,15 @@ function finalIds(input) { value.filter(info => input.ignoreVersions.test(info.version)) .length - input.minVersionsToKeep; + value = value.filter(info => !input.ignoreVersions.test(info.version)); console.log(`toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}`); if (toDelete > input.numOldVersionsToDelete) { //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions input.numOldVersionsToDelete = input.numOldVersionsToDelete + value.length; return toDelete - input.numOldVersionsToDelete >= 0 - ? value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) + ? value.map(info => info.id) : value - .filter(info => !input.ignoreVersions.test(info.version)) .map(info => info.id) .slice(0, toDelete - input.numOldVersionsToDelete); } diff --git a/src/delete.ts b/src/delete.ts index 8684c9f..4c0d85e 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -107,6 +107,7 @@ export function finalIds(input: Input): Observable { value.filter(info => input.ignoreVersions.test(info.version)) .length - input.minVersionsToKeep + value = value.filter(info => !input.ignoreVersions.test(info.version)) console.log( `toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}` ) @@ -115,11 +116,8 @@ export function finalIds(input: Input): Observable { input.numOldVersionsToDelete = input.numOldVersionsToDelete + value.length return toDelete - input.numOldVersionsToDelete >= 0 - ? value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) + ? value.map(info => info.id) : value - .filter(info => !input.ignoreVersions.test(info.version)) .map(info => info.id) .slice(0, toDelete - input.numOldVersionsToDelete) } else return [] From a22ff2a7c59a73a2a9841f40c8b071c63ffeb65a Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 16 Dec 2021 07:02:21 +0000 Subject: [PATCH 08/45] add delete limit of 100 --- __tests__/version/graphql.mock.ts | 2 +- dist/index.js | 10 +++++++--- src/delete.ts | 26 ++++++++++---------------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/__tests__/version/graphql.mock.ts b/__tests__/version/graphql.mock.ts index 4270a68..0883354 100644 --- a/__tests__/version/graphql.mock.ts +++ b/__tests__/version/graphql.mock.ts @@ -6,7 +6,7 @@ import { import * as Graphql from '../../src/version/graphql' import {GetVersionsQueryResponse} from '../../src/version' -export function getMockedOldestQueryResponse( +export function getMockedOldestQueryResponseForLessThan100( numVersions: number ): GetVersionsQueryResponse { const versions = [] diff --git a/dist/index.js b/dist/index.js index 7905c9c..9867c58 100644 --- a/dist/index.js +++ b/dist/index.js @@ -58,16 +58,20 @@ function finalIds(input) { console.log(`in min versions to keep`); return getVersionIds(input.owner, input.repo, input.packageName, 100, input.ignoreVersions, '', input.token).pipe(operators_1.map(value => { console.log(`point 1`); - const toDelete = totalCount - + let toDelete = totalCount - value.filter(info => input.ignoreVersions.test(info.version)) .length - input.minVersionsToKeep; + toDelete = toDelete > 100 ? 100 : toDelete; value = value.filter(info => !input.ignoreVersions.test(info.version)); console.log(`toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}`); - if (toDelete > input.numOldVersionsToDelete) { + if (toDelete > input.numOldVersionsToDelete && + input.numOldVersionsToDelete < 100) { //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions input.numOldVersionsToDelete = - input.numOldVersionsToDelete + value.length; + input.numOldVersionsToDelete + value.length > 100 + ? 100 + : input.numOldVersionsToDelete + value.length; return toDelete - input.numOldVersionsToDelete >= 0 ? value.map(info => info.id) : value diff --git a/src/delete.ts b/src/delete.ts index 4c0d85e..9c44425 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -1,20 +1,8 @@ import {Input} from './input' import {EMPTY, Observable, of, throwError} from 'rxjs' -import {deletePackageVersions, getOldestVersions} from './version' +import {deletePackageVersions, getOldestVersions, VersionInfo} from './version' import {concatMap, map, expand, tap} from 'rxjs/operators' -export interface VersionInfo { - id: string - version: string -} - -export interface QueryInfo { - versions: VersionInfo[] - cursor: string - paginate: boolean - totalCount: number -} - let totalCount: number export function getVersionIds( @@ -102,19 +90,25 @@ export function finalIds(input: Input): Observable { ).pipe( map(value => { console.log(`point 1`) - const toDelete = + let toDelete = totalCount - value.filter(info => input.ignoreVersions.test(info.version)) .length - input.minVersionsToKeep + toDelete = toDelete > 100 ? 100 : toDelete value = value.filter(info => !input.ignoreVersions.test(info.version)) console.log( `toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}` ) - if (toDelete > input.numOldVersionsToDelete) { + if ( + toDelete > input.numOldVersionsToDelete && + input.numOldVersionsToDelete < 100 + ) { //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions input.numOldVersionsToDelete = - input.numOldVersionsToDelete + value.length + input.numOldVersionsToDelete + value.length > 100 + ? 100 + : input.numOldVersionsToDelete + value.length return toDelete - input.numOldVersionsToDelete >= 0 ? value.map(info => info.id) : value From 36894fd813ac8424cee4ecb6ade5d1426fb75206 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 16 Dec 2021 08:54:07 +0000 Subject: [PATCH 09/45] add logs --- dist/index.js | 23 ++++++++++++----------- src/delete.ts | 29 ++++++++++++++--------------- src/input.ts | 4 ++-- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/dist/index.js b/dist/index.js index 9867c58..3e65df9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -48,6 +48,7 @@ function finalIds(input) { ? 0 : input.numOldVersionsToDelete - value.length; console.log(`temp: ${temp} numVersions: ${input.numOldVersionsToDelete} ignore-versions: ${input.ignoreVersions}`); + input.numDeleted += value.filter(info => !input.ignoreVersions.test(info.version)).length; return value .filter(info => !input.ignoreVersions.test(info.version)) .map(info => info.id) @@ -64,19 +65,16 @@ function finalIds(input) { input.minVersionsToKeep; toDelete = toDelete > 100 ? 100 : toDelete; value = value.filter(info => !input.ignoreVersions.test(info.version)); - console.log(`toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}`); - if (toDelete > input.numOldVersionsToDelete && - input.numOldVersionsToDelete < 100) { + console.log(`toDelete: ${toDelete} numVersions: ${input.numDeleted} total count: ${totalCount}`); + if (toDelete > input.numDeleted && input.numDeleted < 100) { //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions - input.numOldVersionsToDelete = - input.numOldVersionsToDelete + value.length > 100 + input.numDeleted = + input.numDeleted + value.length > 100 ? 100 - : input.numOldVersionsToDelete + value.length; - return toDelete - input.numOldVersionsToDelete >= 0 + : input.numDeleted + value.length; + return toDelete - input.numDeleted >= 0 ? value.map(info => info.id) - : value - .map(info => info.id) - .slice(0, toDelete - input.numOldVersionsToDelete); + : value.map(info => info.id).slice(0, toDelete - input.numDeleted); } else return []; @@ -97,7 +95,9 @@ function deleteVersions(input) { console.log('Number of old versions to delete input is 0 or less, no versions will be deleted'); return rxjs_1.of(true); } - return finalIds(input).pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); + const result = finalIds(input); + console.log(`${input.numDeleted} versions deleted`); + return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); } exports.deleteVersions = deleteVersions; @@ -134,6 +134,7 @@ class Input { this.ignoreVersions = validatedParams.ignoreVersions; this.deletePreReleaseVersions = validatedParams.deletePreReleaseVersions; this.token = validatedParams.token; + this.numDeleted = 0; } hasOldestVersionQueryInfo() { return !!(this.owner && diff --git a/src/delete.ts b/src/delete.ts index 9c44425..cc44a95 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -71,6 +71,9 @@ export function finalIds(input: Input): Observable { console.log( `temp: ${temp} numVersions: ${input.numOldVersionsToDelete} ignore-versions: ${input.ignoreVersions}` ) + input.numDeleted += value.filter( + info => !input.ignoreVersions.test(info.version) + ).length return value .filter(info => !input.ignoreVersions.test(info.version)) .map(info => info.id) @@ -98,22 +101,17 @@ export function finalIds(input: Input): Observable { toDelete = toDelete > 100 ? 100 : toDelete value = value.filter(info => !input.ignoreVersions.test(info.version)) console.log( - `toDelete: ${toDelete} numVersions: ${input.numOldVersionsToDelete} total count: ${totalCount}` + `toDelete: ${toDelete} numVersions: ${input.numDeleted} total count: ${totalCount}` ) - if ( - toDelete > input.numOldVersionsToDelete && - input.numOldVersionsToDelete < 100 - ) { + if (toDelete > input.numDeleted && input.numDeleted < 100) { //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions - input.numOldVersionsToDelete = - input.numOldVersionsToDelete + value.length > 100 + input.numDeleted = + input.numDeleted + value.length > 100 ? 100 - : input.numOldVersionsToDelete + value.length - return toDelete - input.numOldVersionsToDelete >= 0 + : input.numDeleted + value.length + return toDelete - input.numDeleted >= 0 ? value.map(info => info.id) - : value - .map(info => info.id) - .slice(0, toDelete - input.numOldVersionsToDelete) + : value.map(info => info.id).slice(0, toDelete - input.numDeleted) } else return [] }) ) @@ -141,7 +139,8 @@ export function deleteVersions(input: Input): Observable { return of(true) } - return finalIds(input).pipe( - concatMap(ids => deletePackageVersions(ids, input.token)) - ) + const result = finalIds(input) + console.log(`${input.numDeleted} versions deleted`) + + return result.pipe(concatMap(ids => deletePackageVersions(ids, input.token))) } diff --git a/src/input.ts b/src/input.ts index dd996ca..35244d5 100644 --- a/src/input.ts +++ b/src/input.ts @@ -1,5 +1,3 @@ -import {throwError} from 'rxjs' - export interface InputParams { packageVersionIds?: string[] owner?: string @@ -34,6 +32,7 @@ export class Input { ignoreVersions: RegExp deletePreReleaseVersions: string token: string + numDeleted: number constructor(params?: InputParams) { const validatedParams: Required = {...defaultParams, ...params} @@ -47,6 +46,7 @@ export class Input { this.ignoreVersions = validatedParams.ignoreVersions this.deletePreReleaseVersions = validatedParams.deletePreReleaseVersions this.token = validatedParams.token + this.numDeleted = 0 } hasOldestVersionQueryInfo(): boolean { From 263c5d855adc57bb2e3db42039bdd6e6dd7b9816 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 08:37:14 +0000 Subject: [PATCH 10/45] logs to check rate limiting --- dist/index.js | 23 ++++++++--------------- src/delete.ts | 16 +--------------- src/version/get-versions.ts | 8 +------- 3 files changed, 10 insertions(+), 37 deletions(-) diff --git a/dist/index.js b/dist/index.js index 3e65df9..209a72b 100644 --- a/dist/index.js +++ b/dist/index.js @@ -28,9 +28,9 @@ const rxjs_1 = __nccwpck_require__(5805); const version_1 = __nccwpck_require__(4428); const operators_1 = __nccwpck_require__(7801); let totalCount; -function getVersionIds(owner, repo, packageName, numVersions, ignoreVersions, cursor, token) { - return version_1.getOldestVersions(owner, repo, packageName, numVersions, ignoreVersions, cursor, token).pipe(operators_1.expand(value => value.paginate - ? version_1.getOldestVersions(owner, repo, packageName, numVersions, ignoreVersions, value.cursor, token) +function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { + return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate + ? version_1.getOldestVersions(owner, repo, packageName, numVersions, value.cursor, token) : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.map(value => value.versions), operators_1.tap(value => value.map(info => console.log(`id0: ${info.id}, version: ${info.version}`)))); } exports.getVersionIds = getVersionIds; @@ -40,14 +40,12 @@ function finalIds(input) { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { - console.log(`in numOldVersionsToDelete`); - return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, input.ignoreVersions, '', input.token).pipe(operators_1.map(value => { + return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length; - console.log(`temp: ${temp} numVersions: ${input.numOldVersionsToDelete} ignore-versions: ${input.ignoreVersions}`); input.numDeleted += value.filter(info => !input.ignoreVersions.test(info.version)).length; return value .filter(info => !input.ignoreVersions.test(info.version)) @@ -56,18 +54,15 @@ function finalIds(input) { })); } else { - console.log(`in min versions to keep`); - return getVersionIds(input.owner, input.repo, input.packageName, 100, input.ignoreVersions, '', input.token).pipe(operators_1.map(value => { - console.log(`point 1`); + return getVersionIds(input.owner, input.repo, input.packageName, 100, '', input.token).pipe(operators_1.map(value => { let toDelete = totalCount - value.filter(info => input.ignoreVersions.test(info.version)) .length - input.minVersionsToKeep; toDelete = toDelete > 100 ? 100 : toDelete; value = value.filter(info => !input.ignoreVersions.test(info.version)); - console.log(`toDelete: ${toDelete} numVersions: ${input.numDeleted} total count: ${totalCount}`); if (toDelete > input.numDeleted && input.numDeleted < 100) { - //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions + // using input.numDeleted to keep track of deleted and remaining packages input.numDeleted = input.numDeleted + value.length > 100 ? 100 @@ -286,6 +281,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs } })).pipe(operators_1.catchError((err) => { const msg = 'query for oldest version failed.'; + console.log(`numversions: ${numVersions} startCursor: ${startCursor}`); return rxjs_1.throwError(err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` : `${msg} verify input parameters are correct`); @@ -310,7 +306,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs } } exports.queryForOldestVersions = queryForOldestVersions; -function getOldestVersions(owner, repo, packageName, numVersions, ignoreVersions, startCursor, token) { +function getOldestVersions(owner, repo, packageName, numVersions, startCursor, token) { return queryForOldestVersions(owner, repo, packageName, numVersions, startCursor, token).pipe(operators_1.map(result => { let r; if (result.repository.packages.edges.length < 1) { @@ -326,9 +322,6 @@ function getOldestVersions(owner, repo, packageName, numVersions, ignoreVersions const versions = result.repository.packages.edges[0].node.versions.edges; const pages = result.repository.packages.edges[0].node.versions.pageInfo; const count = result.repository.packages.edges[0].node.versions.totalCount; - if (versions.length !== numVersions) { - console.log(`number of versions requested was: ${numVersions}, but found: ${versions.length}`); - } r = { versions: versions .map(value => ({ id: value.node.id, version: value.node.version })) diff --git a/src/delete.ts b/src/delete.ts index cc44a95..1633372 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -10,7 +10,6 @@ export function getVersionIds( repo: string, packageName: string, numVersions: number, - ignoreVersions: RegExp, cursor: string, token: string ): Observable { @@ -19,7 +18,6 @@ export function getVersionIds( repo, packageName, numVersions, - ignoreVersions, cursor, token ).pipe( @@ -30,7 +28,6 @@ export function getVersionIds( repo, packageName, numVersions, - ignoreVersions, value.cursor, token ) @@ -52,13 +49,11 @@ export function finalIds(input: Input): Observable { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { - console.log(`in numOldVersionsToDelete`) return getVersionIds( input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, - input.ignoreVersions, '', input.token ).pipe( @@ -68,9 +63,6 @@ export function finalIds(input: Input): Observable { input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length - console.log( - `temp: ${temp} numVersions: ${input.numOldVersionsToDelete} ignore-versions: ${input.ignoreVersions}` - ) input.numDeleted += value.filter( info => !input.ignoreVersions.test(info.version) ).length @@ -81,18 +73,15 @@ export function finalIds(input: Input): Observable { }) ) } else { - console.log(`in min versions to keep`) return getVersionIds( input.owner, input.repo, input.packageName, 100, - input.ignoreVersions, '', input.token ).pipe( map(value => { - console.log(`point 1`) let toDelete = totalCount - value.filter(info => input.ignoreVersions.test(info.version)) @@ -100,11 +89,8 @@ export function finalIds(input: Input): Observable { input.minVersionsToKeep toDelete = toDelete > 100 ? 100 : toDelete value = value.filter(info => !input.ignoreVersions.test(info.version)) - console.log( - `toDelete: ${toDelete} numVersions: ${input.numDeleted} total count: ${totalCount}` - ) if (toDelete > input.numDeleted && input.numDeleted < 100) { - //here input.numOldVersionsToDelete will never have user value hence using it to keep track of deleted versions + // using input.numDeleted to keep track of deleted and remaining packages input.numDeleted = input.numDeleted + value.length > 100 ? 100 diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index 099556f..5d91e6d 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -109,6 +109,7 @@ export function queryForOldestVersions( ).pipe( catchError((err: GraphQlQueryResponse) => { const msg = 'query for oldest version failed.' + console.log(`numversions: ${numVersions} startCursor: ${startCursor}`) return throwError( err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` @@ -146,7 +147,6 @@ export function getOldestVersions( repo: string, packageName: string, numVersions: number, - ignoreVersions: RegExp, startCursor: string, token: string ): Observable { @@ -177,12 +177,6 @@ export function getOldestVersions( const pages = result.repository.packages.edges[0].node.versions.pageInfo const count = result.repository.packages.edges[0].node.versions.totalCount - if (versions.length !== numVersions) { - console.log( - `number of versions requested was: ${numVersions}, but found: ${versions.length}` - ) - } - r = { versions: versions .map(value => ({id: value.node.id, version: value.node.version})) From 5003595d8e94b56a70a29ddb469dd7ae4c39f79a Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 08:54:55 +0000 Subject: [PATCH 11/45] fix delete rate throttling --- dist/index.js | 2 +- src/delete.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index 209a72b..d198f6a 100644 --- a/dist/index.js +++ b/dist/index.js @@ -92,7 +92,7 @@ function deleteVersions(input) { } const result = finalIds(input); console.log(`${input.numDeleted} versions deleted`); - return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); + return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 101), input.token))); } exports.deleteVersions = deleteVersions; diff --git a/src/delete.ts b/src/delete.ts index 1633372..4edc0db 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -128,5 +128,7 @@ export function deleteVersions(input: Input): Observable { const result = finalIds(input) console.log(`${input.numDeleted} versions deleted`) - return result.pipe(concatMap(ids => deletePackageVersions(ids, input.token))) + return result.pipe( + concatMap(ids => deletePackageVersions(ids.slice(0, 101), input.token)) + ) } From 7d96e1e541e37a2e6633b31c0df41c45b4cb2add Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 09:05:08 +0000 Subject: [PATCH 12/45] fix delete rate throttling --- dist/index.js | 2 +- src/delete.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index d198f6a..662b781 100644 --- a/dist/index.js +++ b/dist/index.js @@ -92,7 +92,7 @@ function deleteVersions(input) { } const result = finalIds(input); console.log(`${input.numDeleted} versions deleted`); - return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 101), input.token))); + return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 100), input.token))); } exports.deleteVersions = deleteVersions; diff --git a/src/delete.ts b/src/delete.ts index 4edc0db..d19f32a 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -129,6 +129,6 @@ export function deleteVersions(input: Input): Observable { console.log(`${input.numDeleted} versions deleted`) return result.pipe( - concatMap(ids => deletePackageVersions(ids.slice(0, 101), input.token)) + concatMap(ids => deletePackageVersions(ids.slice(0, 100), input.token)) ) } From 04947a51f37790e8ccd380e75f8d5207182c45ec Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 09:21:35 +0000 Subject: [PATCH 13/45] fix logs --- dist/index.js | 6 +++--- src/delete.ts | 12 +++++------- src/version/delete-version.ts | 1 - 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/dist/index.js b/dist/index.js index 662b781..cbf273b 100644 --- a/dist/index.js +++ b/dist/index.js @@ -31,7 +31,7 @@ let totalCount; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate ? version_1.getOldestVersions(owner, repo, packageName, numVersions, value.cursor, token) - : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.map(value => value.versions), operators_1.tap(value => value.map(info => console.log(`id0: ${info.id}, version: ${info.version}`)))); + : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.map(value => value.versions)); } exports.getVersionIds = getVersionIds; function finalIds(input) { @@ -91,7 +91,8 @@ function deleteVersions(input) { return rxjs_1.of(true); } const result = finalIds(input); - console.log(`${input.numDeleted} versions deleted`); + result.pipe(operators_1.tap(value => (input.numDeleted = value.length < 100 ? value.length : 100))); + console.log(`${input.numDeleted} versions will be deleted`); return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 100), input.token))); } exports.deleteVersions = deleteVersions; @@ -191,7 +192,6 @@ function deletePackageVersion(packageVersionId, token) { exports.deletePackageVersion = deletePackageVersion; function deletePackageVersions(packageVersionIds, token) { if (packageVersionIds.length === 0) { - console.log('no package version ids found, no versions will be deleted'); return rxjs_1.of(true); } const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { diff --git a/src/delete.ts b/src/delete.ts index d19f32a..0721eb5 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -34,12 +34,7 @@ export function getVersionIds( : EMPTY ), tap(value => (totalCount = value.totalCount)), - map(value => value.versions), - tap(value => - value.map(info => - console.log(`id0: ${info.id}, version: ${info.version}`) - ) - ) + map(value => value.versions) ) } @@ -126,7 +121,10 @@ export function deleteVersions(input: Input): Observable { } const result = finalIds(input) - console.log(`${input.numDeleted} versions deleted`) + result.pipe( + tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)) + ) + console.log(`${input.numDeleted} versions will be deleted`) return result.pipe( concatMap(ids => deletePackageVersions(ids.slice(0, 100), input.token)) diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 44508e9..40a7269 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -45,7 +45,6 @@ export function deletePackageVersions( token: string ): Observable { if (packageVersionIds.length === 0) { - console.log('no package version ids found, no versions will be deleted') return of(true) } From 106b04e07645745387b1fdf214b54e2d8da31302 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 09:31:36 +0000 Subject: [PATCH 14/45] fix logs --- dist/index.js | 4 +--- src/delete.ts | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dist/index.js b/dist/index.js index cbf273b..a683779 100644 --- a/dist/index.js +++ b/dist/index.js @@ -91,9 +91,7 @@ function deleteVersions(input) { return rxjs_1.of(true); } const result = finalIds(input); - result.pipe(operators_1.tap(value => (input.numDeleted = value.length < 100 ? value.length : 100))); - console.log(`${input.numDeleted} versions will be deleted`); - return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 100), input.token))); + return result.pipe(operators_1.tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), operators_1.tap(() => console.log(`${input.numDeleted} versions will be deleted`)), operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 100), input.token))); } exports.deleteVersions = deleteVersions; diff --git a/src/delete.ts b/src/delete.ts index 0721eb5..ca47b58 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -121,12 +121,10 @@ export function deleteVersions(input: Input): Observable { } const result = finalIds(input) - result.pipe( - tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)) - ) - console.log(`${input.numDeleted} versions will be deleted`) return result.pipe( + tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), + tap(() => console.log(`${input.numDeleted} versions will be deleted`)), concatMap(ids => deletePackageVersions(ids.slice(0, 100), input.token)) ) } From ba58e0e966649e35082c4062a1c1fcf5f4c42941 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 09:45:54 +0000 Subject: [PATCH 15/45] update ReadME --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9d83333..7b9e94b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Delete Package Versions -This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages). +This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages). This action will only delete a maximum of 100 versions in one run. ### What It Can Do @@ -42,14 +42,12 @@ This action deletes versions of a package from [GitHub Packages](https://github. # The number of old versions to delete starting from the oldest version. # Defaults to 1. - # Cannot be more than 100. num-old-versions-to-delete: # The number of latest versions to not delete. - # Defaults to 0. - # When this is set greater than 0 it will delete all deletable package versions except the specified no. - # `num-old-versions-to-delete` will not be taken into account with this option. - # Cannot be more than 100. + # This cannot be specified with `num-old-versions-to-delete`. By default, `num-old-versions-to-delete` is taken into account. + # When set to 0, all deletable versions will be deleted. + # When set greater than 0, all deletable package versions except the specified number will be deleted. min-versions-to-keep: # The package versions to exclude from deletion. @@ -61,6 +59,7 @@ This action deletes versions of a package from [GitHub Packages](https://github. # The number of pre-release versions to keep can be set by using `min-versions-to-keep` value with this. # When `min-versions-to-keep` is 0, all pre-release versions get deleted. # Defaults to false. + # Cannot be used with `num-old-versions-to-delete`. delete-only-pre-release-versions: # The token used to authenticate with GitHub Packages. From 370a3503533ef9581121d8b9aec28a9683bc8f7f Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 10:00:12 +0000 Subject: [PATCH 16/45] update ReadME --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7b9e94b..55a3d27 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,6 @@ This action deletes versions of a package from [GitHub Packages](https://github. * Delete version(s) of a package that is hosted in a different repo than the one executing the workflow * Delete a single version * Delete multiple versions - - * Delete specific version(s) # Usage @@ -70,6 +68,18 @@ This action deletes versions of a package from [GitHub Packages](https://github. token: ``` +# Valid Input Combinations + +`owner`, `repo`, `package-name` and `token` can be used with the following combinations in a workflow - + + - `num-old-versions-to-delete` + - `min-versions-to-keep` + - `delete-only-pre-release-versions` + - `ignore-versions` + - `num-old-versions-to-delete` + `ignore-versions` + - `min-versions-to-keep` + `ignore-versions` + - `min-versions-to-keep` + `delete-only-pre-release-versions` + # Scenarios - [Delete all pre-release versions except y latest pre-release package versions](#delete-all-pre-release-versions-except-y-latest-pre-release-package-versions) From b7877514c6c9e4197b66e4f782065f12e8415733 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 15:34:15 +0530 Subject: [PATCH 17/45] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55a3d27..8e1050d 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ This action deletes versions of a package from [GitHub Packages](https://github. # The number of pre-release versions to keep can be set by using `min-versions-to-keep` value with this. # When `min-versions-to-keep` is 0, all pre-release versions get deleted. # Defaults to false. - # Cannot be used with `num-old-versions-to-delete`. + # Cannot be used with `num-old-versions-to-delete` and `ignore-versions`. delete-only-pre-release-versions: # The token used to authenticate with GitHub Packages. From 94d12027263a888d1c0bd92f7c26fd2fff945a8c Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Mon, 20 Dec 2021 16:17:20 +0530 Subject: [PATCH 18/45] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e1050d..aa4bef3 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ This action deletes versions of a package from [GitHub Packages](https://github. num-old-versions-to-delete: # The number of latest versions to not delete. - # This cannot be specified with `num-old-versions-to-delete`. By default, `num-old-versions-to-delete` is taken into account. + # This cannot be specified with `num-old-versions-to-delete`. By default, `num-old-versions-to-delete` takes precedence over `min-versions-to-keep`. # When set to 0, all deletable versions will be deleted. # When set greater than 0, all deletable package versions except the specified number will be deleted. min-versions-to-keep: From d846be5128e29a5e0be82504033990775c5fb1bc Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 21 Dec 2021 09:29:12 +0000 Subject: [PATCH 19/45] add unit tests --- __tests__/delete.test.ts | 53 +++++++++++++-------------- __tests__/version/get-version.test.ts | 30 +++++++++------ __tests__/version/graphql.mock.ts | 24 +++++++----- 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/__tests__/delete.test.ts b/__tests__/delete.test.ts index 41e42ef..a8208c7 100644 --- a/__tests__/delete.test.ts +++ b/__tests__/delete.test.ts @@ -1,27 +1,23 @@ import {Input, InputParams} from '../src/input' -import {deleteVersions, getVersionIds} from '../src/delete' +import {deleteVersions, finalIds} from '../src/delete' -describe.skip('index tests -- call graphql', () => { +describe('index tests -- call graphql', () => { it('getVersionIds test -- get oldest version', done => { const numVersions = 1 - getVersionIds(getInput({numOldVersionsToDelete: numVersions})).subscribe( - ids => { - expect(ids.length).toBeLessThanOrEqual(numVersions) - done() - } - ) + finalIds(getInput({numOldVersionsToDelete: numVersions})).subscribe(ids => { + expect(ids.length).toBeGreaterThanOrEqual(numVersions) + done() + }) }) it('getVersionIds test -- get oldest 3 versions', done => { const numVersions = 3 - getVersionIds(getInput({numOldVersionsToDelete: numVersions})).subscribe( - ids => { - expect(ids.length).toBeLessThanOrEqual(numVersions) - done() - } - ) + finalIds(getInput({numOldVersionsToDelete: numVersions})).subscribe(ids => { + expect(ids.length).toBeGreaterThanOrEqual(numVersions) + done() + }) }) it('getVersionIds test -- supplied package version id', done => { @@ -31,7 +27,7 @@ describe.skip('index tests -- call graphql', () => { 'CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC' ] - getVersionIds(getInput({packageVersionIds: suppliedIds})).subscribe(ids => { + finalIds(getInput({packageVersionIds: suppliedIds})).subscribe(ids => { expect(ids).toBe(suppliedIds) done() }) @@ -58,24 +54,24 @@ describe.skip('index tests -- call graphql', () => { }) it.skip('deleteVersions test -- delete oldest version', done => { - deleteVersions( - getInput({numOldVersionsToDelete: 2, minVersionsToKeep: 1}) - ).subscribe(isSuccess => { - expect(isSuccess).toBe(true) - done() - }) + deleteVersions(getInput({numOldVersionsToDelete: 1})).subscribe( + isSuccess => { + expect(isSuccess).toBe(true) + done() + } + ) }) it.skip('deleteVersions test -- delete 3 oldest versions', done => { - deleteVersions( - getInput({numOldVersionsToDelete: 3, minVersionsToKeep: 1}) - ).subscribe(isSuccess => { - expect(isSuccess).toBe(true) - done() - }) + deleteVersions(getInput({numOldVersionsToDelete: 3})).subscribe( + isSuccess => { + expect(isSuccess).toBe(true) + done() + } + ) }) - it('deleteVersions test -- keep 5 versions', done => { + it.skip('deleteVersions test -- keep 5 versions', done => { deleteVersions(getInput({minVersionsToKeep: 5})).subscribe(isSuccess => { expect(isSuccess).toBe(true) done() @@ -90,6 +86,7 @@ const defaultInput: InputParams = { packageName: 'onlypkg.maven', numOldVersionsToDelete: 1, minVersionsToKeep: 1, + ignoreVersions: RegExp('^$'), token: process.env.GITHUB_TOKEN as string } diff --git a/__tests__/version/get-version.test.ts b/__tests__/version/get-version.test.ts index 481a96d..d3a2d72 100644 --- a/__tests__/version/get-version.test.ts +++ b/__tests__/version/get-version.test.ts @@ -1,18 +1,23 @@ -// eslint-disable-next-line @typescript-eslint/ban-ts-ignore -// @ts-ignore import {mockOldestQueryResponse} from './graphql.mock' import { getOldestVersions as _getOldestVersions, - VersionInfo + QueryInfo } from '../../src/version' import {Observable} from 'rxjs' -describe.skip('get versions tests -- call graphql', () => { +describe('get versions tests -- call graphql', () => { it('getOldestVersions -- succeeds', done => { const numVersions = 1 + getOldestVersions({numVersions}).subscribe(result => { + expect(result.versions.length).toBe(numVersions) + done() + }) + }) - getOldestVersions({numVersions}).subscribe(versions => { - expect(versions.length).toBe(numVersions) + it('getOldestVersions -- succeeds for more than 100 versions', done => { + const numVersions = 110 + getOldestVersions({numVersions}).subscribe(result => { + expect(result.versions.length).toBe(100) done() }) }) @@ -33,8 +38,8 @@ describe('get versions tests -- mock graphql', () => { const numVersions = 5 mockOldestQueryResponse(numVersions) - getOldestVersions({numVersions}).subscribe(versions => { - expect(versions.length).toBe(numVersions) + getOldestVersions({numVersions}).subscribe(result => { + expect(result.versions.length).toBe(numVersions) done() }) }) @@ -45,24 +50,27 @@ interface Params { repo?: string packageName?: string numVersions?: number + startCursor?: string token?: string } const defaultParams = { owner: 'namratajha', repo: 'only-pkg', - packageName: 'onlypkg.maven', - numVersions: 3, + packageName: 'only-pkg', + numVersions: 1, + startCursor: '', token: process.env.GITHUB_TOKEN as string } -function getOldestVersions(params?: Params): Observable { +function getOldestVersions(params?: Params): Observable { const p: Required = {...defaultParams, ...params} return _getOldestVersions( p.owner, p.repo, p.packageName, p.numVersions, + p.startCursor, p.token ) } diff --git a/__tests__/version/graphql.mock.ts b/__tests__/version/graphql.mock.ts index 0883354..c2fb04e 100644 --- a/__tests__/version/graphql.mock.ts +++ b/__tests__/version/graphql.mock.ts @@ -6,11 +6,11 @@ import { import * as Graphql from '../../src/version/graphql' import {GetVersionsQueryResponse} from '../../src/version' -export function getMockedOldestQueryResponseForLessThan100( +export function getMockedOldestQueryResponse( numVersions: number ): GetVersionsQueryResponse { const versions = [] - + numVersions = numVersions < 100 ? numVersions : numVersions for (let i = 1; i <= numVersions; ++i) { versions.push({ node: { @@ -28,7 +28,12 @@ export function getMockedOldestQueryResponseForLessThan100( node: { name: 'test', versions: { - edges: versions.reverse() + totalCount: 200, + edges: versions.reverse(), + pageInfo: { + startCursor: 'AAA', + hasPreviousPage: false + } } } } @@ -38,12 +43,13 @@ export function getMockedOldestQueryResponseForLessThan100( } } -export function mockOldestQueryResponse( - numVersions: number -) { - const response = new Promise((resolve) => { +export function mockOldestQueryResponse(numVersions: number): void { + const response = new Promise(resolve => { resolve(getMockedOldestQueryResponse(numVersions)) }) as Promise - jest.spyOn(Graphql, 'graphql').mockImplementation( - (token: string, query: string, parameters: RequestParameters) => response) + jest + .spyOn(Graphql, 'graphql') + .mockImplementation( + (token: string, query: string, parameters: RequestParameters) => response + ) } From 75db2288ca6f67e64b893999b62890b0c5f5dc8c Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 21 Dec 2021 16:19:51 +0000 Subject: [PATCH 20/45] intermitent failed delete mutations logs --- dist/index.js | 6 +++++- src/delete.ts | 5 ++++- src/version/delete-version.ts | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index a683779..fa96525 100644 --- a/dist/index.js +++ b/dist/index.js @@ -91,7 +91,10 @@ function deleteVersions(input) { return rxjs_1.of(true); } const result = finalIds(input); - return result.pipe(operators_1.tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), operators_1.tap(() => console.log(`${input.numDeleted} versions will be deleted`)), operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 100), input.token))); + return result.pipe(operators_1.tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), operators_1.tap(() => { + if (input.numDeleted > 0) + console.log(`${input.numDeleted} versions will be deleted`); + }), operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 100), input.token))); } exports.deleteVersions = deleteVersions; @@ -192,6 +195,7 @@ function deletePackageVersions(packageVersionIds, token) { if (packageVersionIds.length === 0) { return rxjs_1.of(true); } + console.log(`in delete length: ${packageVersionIds.length}`); const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { if (result) { console.log(`version with id: ${id}, deleted`); diff --git a/src/delete.ts b/src/delete.ts index ca47b58..148a60f 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -124,7 +124,10 @@ export function deleteVersions(input: Input): Observable { return result.pipe( tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), - tap(() => console.log(`${input.numDeleted} versions will be deleted`)), + tap(() => { + if (input.numDeleted > 0) + console.log(`${input.numDeleted} versions will be deleted`) + }), concatMap(ids => deletePackageVersions(ids.slice(0, 100), input.token)) ) } diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 40a7269..8412c3d 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -48,6 +48,7 @@ export function deletePackageVersions( return of(true) } + console.log(`in delete length: ${packageVersionIds.length}`) const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe( tap(result => { From c8087bcc700995f151a1ff38b699fe67538e0f67 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 21 Dec 2021 17:34:02 +0000 Subject: [PATCH 21/45] fix failed delete mutations --- dist/index.js | 33 ++++++++++++++++++--------------- src/delete.ts | 32 ++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/dist/index.js b/dist/index.js index fa96525..3a736ce 100644 --- a/dist/index.js +++ b/dist/index.js @@ -31,7 +31,7 @@ let totalCount; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate ? version_1.getOldestVersions(owner, repo, packageName, numVersions, value.cursor, token) - : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.map(value => value.versions)); + : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.tap(value => console.log(`in expand value length: ${value.versions.length}`)), operators_1.map(value => value.versions)); } exports.getVersionIds = getVersionIds; function finalIds(input) { @@ -40,6 +40,8 @@ function finalIds(input) { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + input.numOldVersionsToDelete = + input.numOldVersionsToDelete < 100 ? input.numOldVersionsToDelete : 100; return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = @@ -55,21 +57,22 @@ function finalIds(input) { } else { return getVersionIds(input.owner, input.repo, input.packageName, 100, '', input.token).pipe(operators_1.map(value => { - let toDelete = totalCount - - value.filter(info => input.ignoreVersions.test(info.version)) - .length - - input.minVersionsToKeep; - toDelete = toDelete > 100 ? 100 : toDelete; + totalCount = + totalCount - + value.filter(info => input.ignoreVersions.test(info.version)).length; value = value.filter(info => !input.ignoreVersions.test(info.version)); - if (toDelete > input.numDeleted && input.numDeleted < 100) { + let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted; + toDelete = toDelete > value.length ? value.length : toDelete; + if (toDelete > 0 && input.numDeleted < 100) { // using input.numDeleted to keep track of deleted and remaining packages - input.numDeleted = - input.numDeleted + value.length > 100 - ? 100 - : input.numDeleted + value.length; - return toDelete - input.numDeleted >= 0 - ? value.map(info => info.id) - : value.map(info => info.id).slice(0, toDelete - input.numDeleted); + if (input.numDeleted + toDelete > 100) { + toDelete = 100 - input.numDeleted; + input.numDeleted = 100; + } + else { + input.numDeleted = input.numDeleted + toDelete; + } + return value.map(info => info.id).slice(0, toDelete); } else return []; @@ -94,7 +97,7 @@ function deleteVersions(input) { return result.pipe(operators_1.tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), operators_1.tap(() => { if (input.numDeleted > 0) console.log(`${input.numDeleted} versions will be deleted`); - }), operators_1.concatMap(ids => version_1.deletePackageVersions(ids.slice(0, 100), input.token))); + }), operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); } exports.deleteVersions = deleteVersions; diff --git a/src/delete.ts b/src/delete.ts index 148a60f..b5cc3a1 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -34,6 +34,9 @@ export function getVersionIds( : EMPTY ), tap(value => (totalCount = value.totalCount)), + tap(value => + console.log(`in expand value length: ${value.versions.length}`) + ), map(value => value.versions) ) } @@ -44,6 +47,8 @@ export function finalIds(input: Input): Observable { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + input.numOldVersionsToDelete = + input.numOldVersionsToDelete < 100 ? input.numOldVersionsToDelete : 100 return getVersionIds( input.owner, input.repo, @@ -77,22 +82,21 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { - let toDelete = + totalCount = totalCount - - value.filter(info => input.ignoreVersions.test(info.version)) - .length - - input.minVersionsToKeep - toDelete = toDelete > 100 ? 100 : toDelete + value.filter(info => input.ignoreVersions.test(info.version)).length value = value.filter(info => !input.ignoreVersions.test(info.version)) - if (toDelete > input.numDeleted && input.numDeleted < 100) { + let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted + toDelete = toDelete > value.length ? value.length : toDelete + if (toDelete > 0 && input.numDeleted < 100) { // using input.numDeleted to keep track of deleted and remaining packages - input.numDeleted = - input.numDeleted + value.length > 100 - ? 100 - : input.numDeleted + value.length - return toDelete - input.numDeleted >= 0 - ? value.map(info => info.id) - : value.map(info => info.id).slice(0, toDelete - input.numDeleted) + if (input.numDeleted + toDelete > 100) { + toDelete = 100 - input.numDeleted + input.numDeleted = 100 + } else { + input.numDeleted = input.numDeleted + toDelete + } + return value.map(info => info.id).slice(0, toDelete) } else return [] }) ) @@ -128,6 +132,6 @@ export function deleteVersions(input: Input): Observable { if (input.numDeleted > 0) console.log(`${input.numDeleted} versions will be deleted`) }), - concatMap(ids => deletePackageVersions(ids.slice(0, 100), input.token)) + concatMap(ids => deletePackageVersions(ids, input.token)) ) } From db4d66086aa9d3bd57b1f854b998674ca9846f7d Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 21 Dec 2021 18:14:06 +0000 Subject: [PATCH 22/45] test delete mutation --- dist/index.js | 3 ++- src/delete.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index 3a736ce..ff6d73d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -72,7 +72,8 @@ function finalIds(input) { else { input.numDeleted = input.numDeleted + toDelete; } - return value.map(info => info.id).slice(0, toDelete); + //return value.map(info => info.id).slice(0, toDelete) + return []; } else return []; diff --git a/src/delete.ts b/src/delete.ts index b5cc3a1..87b98e7 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -96,7 +96,8 @@ export function finalIds(input: Input): Observable { } else { input.numDeleted = input.numDeleted + toDelete } - return value.map(info => info.id).slice(0, toDelete) + //return value.map(info => info.id).slice(0, toDelete) + return [] } else return [] }) ) From 21a6c99c5d5ac7fd1ed0e39b84d5e2767c2f42dc Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 21 Dec 2021 18:23:06 +0000 Subject: [PATCH 23/45] rate throttling at 100 --- dist/index.js | 13 ++++++------- src/delete.ts | 12 +++++------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/dist/index.js b/dist/index.js index ff6d73d..87934e6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -63,17 +63,16 @@ function finalIds(input) { value = value.filter(info => !input.ignoreVersions.test(info.version)); let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted; toDelete = toDelete > value.length ? value.length : toDelete; - if (toDelete > 0 && input.numDeleted < 100) { + if (toDelete > 0 && input.numDeleted < 99) { // using input.numDeleted to keep track of deleted and remaining packages - if (input.numDeleted + toDelete > 100) { - toDelete = 100 - input.numDeleted; - input.numDeleted = 100; + if (input.numDeleted + toDelete > 99) { + toDelete = 99 - input.numDeleted; + input.numDeleted = 99; } else { input.numDeleted = input.numDeleted + toDelete; } - //return value.map(info => info.id).slice(0, toDelete) - return []; + return value.map(info => info.id).slice(0, toDelete); } else return []; @@ -95,7 +94,7 @@ function deleteVersions(input) { return rxjs_1.of(true); } const result = finalIds(input); - return result.pipe(operators_1.tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), operators_1.tap(() => { + return result.pipe(operators_1.tap(() => { if (input.numDeleted > 0) console.log(`${input.numDeleted} versions will be deleted`); }), operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); diff --git a/src/delete.ts b/src/delete.ts index 87b98e7..1fa8ce1 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -88,16 +88,15 @@ export function finalIds(input: Input): Observable { value = value.filter(info => !input.ignoreVersions.test(info.version)) let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted toDelete = toDelete > value.length ? value.length : toDelete - if (toDelete > 0 && input.numDeleted < 100) { + if (toDelete > 0 && input.numDeleted < 99) { // using input.numDeleted to keep track of deleted and remaining packages - if (input.numDeleted + toDelete > 100) { - toDelete = 100 - input.numDeleted - input.numDeleted = 100 + if (input.numDeleted + toDelete > 99) { + toDelete = 99 - input.numDeleted + input.numDeleted = 99 } else { input.numDeleted = input.numDeleted + toDelete } - //return value.map(info => info.id).slice(0, toDelete) - return [] + return value.map(info => info.id).slice(0, toDelete) } else return [] }) ) @@ -128,7 +127,6 @@ export function deleteVersions(input: Input): Observable { const result = finalIds(input) return result.pipe( - tap(value => (input.numDeleted = value.length < 100 ? value.length : 100)), tap(() => { if (input.numDeleted > 0) console.log(`${input.numDeleted} versions will be deleted`) From e6a9d9c8022cd2b0277022e30d6f3a28cef8e1e1 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Tue, 21 Dec 2021 18:52:10 +0000 Subject: [PATCH 24/45] set RATE_LIMIT=99 --- dist/index.js | 18 ++++++++++-------- src/delete.ts | 16 ++++++++-------- src/version/delete-version.ts | 1 - src/version/get-versions.ts | 2 +- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/dist/index.js b/dist/index.js index 87934e6..1eb34df 100644 --- a/dist/index.js +++ b/dist/index.js @@ -27,11 +27,12 @@ exports.deleteVersions = exports.finalIds = exports.getVersionIds = void 0; const rxjs_1 = __nccwpck_require__(5805); const version_1 = __nccwpck_require__(4428); const operators_1 = __nccwpck_require__(7801); +const RATE_LIMIT = 99; let totalCount; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate ? version_1.getOldestVersions(owner, repo, packageName, numVersions, value.cursor, token) - : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.tap(value => console.log(`in expand value length: ${value.versions.length}`)), operators_1.map(value => value.versions)); + : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.map(value => value.versions)); } exports.getVersionIds = getVersionIds; function finalIds(input) { @@ -41,7 +42,9 @@ function finalIds(input) { if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { input.numOldVersionsToDelete = - input.numOldVersionsToDelete < 100 ? input.numOldVersionsToDelete : 100; + input.numOldVersionsToDelete < RATE_LIMIT + ? input.numOldVersionsToDelete + : RATE_LIMIT; return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = @@ -56,18 +59,18 @@ function finalIds(input) { })); } else { - return getVersionIds(input.owner, input.repo, input.packageName, 100, '', input.token).pipe(operators_1.map(value => { + return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => { totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length; value = value.filter(info => !input.ignoreVersions.test(info.version)); let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted; toDelete = toDelete > value.length ? value.length : toDelete; - if (toDelete > 0 && input.numDeleted < 99) { + if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { // using input.numDeleted to keep track of deleted and remaining packages if (input.numDeleted + toDelete > 99) { - toDelete = 99 - input.numDeleted; - input.numDeleted = 99; + toDelete = RATE_LIMIT - input.numDeleted; + input.numDeleted = RATE_LIMIT; } else { input.numDeleted = input.numDeleted + toDelete; @@ -198,7 +201,6 @@ function deletePackageVersions(packageVersionIds, token) { if (packageVersionIds.length === 0) { return rxjs_1.of(true); } - console.log(`in delete length: ${packageVersionIds.length}`); const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { if (result) { console.log(`version with id: ${id}, deleted`); @@ -280,7 +282,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs owner, repo, package: packageName, - last: numVersions > 100 ? 100 : numVersions, + last: numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } diff --git a/src/delete.ts b/src/delete.ts index 1fa8ce1..330e2b6 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -3,6 +3,7 @@ import {EMPTY, Observable, of, throwError} from 'rxjs' import {deletePackageVersions, getOldestVersions, VersionInfo} from './version' import {concatMap, map, expand, tap} from 'rxjs/operators' +const RATE_LIMIT = 99 let totalCount: number export function getVersionIds( @@ -34,9 +35,6 @@ export function getVersionIds( : EMPTY ), tap(value => (totalCount = value.totalCount)), - tap(value => - console.log(`in expand value length: ${value.versions.length}`) - ), map(value => value.versions) ) } @@ -48,7 +46,9 @@ export function finalIds(input: Input): Observable { if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { input.numOldVersionsToDelete = - input.numOldVersionsToDelete < 100 ? input.numOldVersionsToDelete : 100 + input.numOldVersionsToDelete < RATE_LIMIT + ? input.numOldVersionsToDelete + : RATE_LIMIT return getVersionIds( input.owner, input.repo, @@ -77,7 +77,7 @@ export function finalIds(input: Input): Observable { input.owner, input.repo, input.packageName, - 100, + RATE_LIMIT, '', input.token ).pipe( @@ -88,11 +88,11 @@ export function finalIds(input: Input): Observable { value = value.filter(info => !input.ignoreVersions.test(info.version)) let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted toDelete = toDelete > value.length ? value.length : toDelete - if (toDelete > 0 && input.numDeleted < 99) { + if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { // using input.numDeleted to keep track of deleted and remaining packages if (input.numDeleted + toDelete > 99) { - toDelete = 99 - input.numDeleted - input.numDeleted = 99 + toDelete = RATE_LIMIT - input.numDeleted + input.numDeleted = RATE_LIMIT } else { input.numDeleted = input.numDeleted + toDelete } diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 8412c3d..40a7269 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -48,7 +48,6 @@ export function deletePackageVersions( return of(true) } - console.log(`in delete length: ${packageVersionIds.length}`) const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe( tap(result => { diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index 5d91e6d..dad6909 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -101,7 +101,7 @@ export function queryForOldestVersions( owner, repo, package: packageName, - last: numVersions > 100 ? 100 : numVersions, + last: numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } From 508f6970a5f85097117972847749c79e0a276b26 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 09:59:18 +0000 Subject: [PATCH 25/45] bypass rate limit --- dist/index.js | 49 +++++++++++++++++++++-------------- src/delete.ts | 28 +++++++++++++------- src/version/delete-version.ts | 10 ++++++- src/version/get-versions.ts | 2 +- 4 files changed, 58 insertions(+), 31 deletions(-) diff --git a/dist/index.js b/dist/index.js index 1eb34df..52ec2ab 100644 --- a/dist/index.js +++ b/dist/index.js @@ -27,7 +27,7 @@ exports.deleteVersions = exports.finalIds = exports.getVersionIds = void 0; const rxjs_1 = __nccwpck_require__(5805); const version_1 = __nccwpck_require__(4428); const operators_1 = __nccwpck_require__(7801); -const RATE_LIMIT = 99; +//const RATE_LIMIT = 99 let totalCount; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate @@ -41,40 +41,44 @@ function finalIds(input) { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + /* input.numOldVersionsToDelete = - input.numOldVersionsToDelete < RATE_LIMIT - ? input.numOldVersionsToDelete - : RATE_LIMIT; + input.numOldVersionsToDelete < RATE_LIMIT + ? input.numOldVersionsToDelete + : RATE_LIMIT*/ + console.log(`input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`); return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { + value = value.filter(info => !input.ignoreVersions.test(info.version)); const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length; - input.numDeleted += value.filter(info => !input.ignoreVersions.test(info.version)).length; - return value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, temp); + console.log(`temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}`); + return value.map(info => info.id).slice(0, temp); })); } else { - return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => { + return getVersionIds(input.owner, input.repo, input.packageName, 100, '', input.token).pipe(operators_1.map(value => { totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length; value = value.filter(info => !input.ignoreVersions.test(info.version)); let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted; toDelete = toDelete > value.length ? value.length : toDelete; + /* if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { - // using input.numDeleted to keep track of deleted and remaining packages - if (input.numDeleted + toDelete > 99) { - toDelete = RATE_LIMIT - input.numDeleted; - input.numDeleted = RATE_LIMIT; - } - else { - input.numDeleted = input.numDeleted + toDelete; - } + // using input.numDeleted to keep track of deleted and remaining packages + if (input.numDeleted + toDelete > 99) { + toDelete = RATE_LIMIT - input.numDeleted + input.numDeleted = RATE_LIMIT + } else { + input.numDeleted = input.numDeleted + toDelete + } + return value.map(info => info.id).slice(0, toDelete) + } else return []*/ + if (toDelete > 0) { + input.numDeleted += toDelete; return value.map(info => info.id).slice(0, toDelete); } else @@ -177,6 +181,7 @@ exports.deletePackageVersions = exports.deletePackageVersion = void 0; const rxjs_1 = __nccwpck_require__(5805); const operators_1 = __nccwpck_require__(7801); const graphql_1 = __nccwpck_require__(6320); +let deleted = 0; const mutation = ` mutation deletePackageVersion($packageVersionId: String!) { deletePackageVersion(input: {packageVersionId: $packageVersionId}) { @@ -184,6 +189,11 @@ const mutation = ` } }`; function deletePackageVersion(packageVersionId, token) { + if (deleted === 99) { + console.log(`reaching rate limit`); + operators_1.delay(5000); + } + deleted += 1; return rxjs_1.from(graphql_1.graphql(token, mutation, { packageVersionId, headers: { @@ -202,6 +212,7 @@ function deletePackageVersions(packageVersionIds, token) { return rxjs_1.of(true); } const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { + console.log(`versions Deleted 0 : ${deleted}`); if (result) { console.log(`version with id: ${id}, deleted`); } @@ -282,7 +293,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs owner, repo, package: packageName, - last: numVersions, + last: numVersions > 100 ? 100 : numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } diff --git a/src/delete.ts b/src/delete.ts index 330e2b6..0466fbf 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -3,7 +3,7 @@ import {EMPTY, Observable, of, throwError} from 'rxjs' import {deletePackageVersions, getOldestVersions, VersionInfo} from './version' import {concatMap, map, expand, tap} from 'rxjs/operators' -const RATE_LIMIT = 99 +//const RATE_LIMIT = 99 let totalCount: number export function getVersionIds( @@ -45,10 +45,14 @@ export function finalIds(input: Input): Observable { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + /* input.numOldVersionsToDelete = input.numOldVersionsToDelete < RATE_LIMIT ? input.numOldVersionsToDelete - : RATE_LIMIT + : RATE_LIMIT*/ + console.log( + `input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}` + ) return getVersionIds( input.owner, input.repo, @@ -58,18 +62,16 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { + value = value.filter(info => !input.ignoreVersions.test(info.version)) const temp = input.numOldVersionsToDelete input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length - input.numDeleted += value.filter( - info => !input.ignoreVersions.test(info.version) - ).length - return value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, temp) + console.log( + `temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}` + ) + return value.map(info => info.id).slice(0, temp) }) ) } else { @@ -77,7 +79,7 @@ export function finalIds(input: Input): Observable { input.owner, input.repo, input.packageName, - RATE_LIMIT, + 100, '', input.token ).pipe( @@ -88,6 +90,7 @@ export function finalIds(input: Input): Observable { value = value.filter(info => !input.ignoreVersions.test(info.version)) let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted toDelete = toDelete > value.length ? value.length : toDelete + /* if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { // using input.numDeleted to keep track of deleted and remaining packages if (input.numDeleted + toDelete > 99) { @@ -97,6 +100,11 @@ export function finalIds(input: Input): Observable { input.numDeleted = input.numDeleted + toDelete } return value.map(info => info.id).slice(0, toDelete) + } else return []*/ + + if (toDelete > 0) { + input.numDeleted += toDelete + return value.map(info => info.id).slice(0, toDelete) } else return [] }) ) diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 40a7269..1e6c079 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -1,8 +1,10 @@ import {from, Observable, merge, throwError, of} from 'rxjs' -import {catchError, map, tap} from 'rxjs/operators' +import {catchError, delay, map, tap} from 'rxjs/operators' import {GraphQlQueryResponse} from '@octokit/graphql/dist-types/types' import {graphql} from './graphql' +let deleted = 0 + export interface DeletePackageVersionMutationResponse { deletePackageVersion: { success: boolean @@ -20,6 +22,11 @@ export function deletePackageVersion( packageVersionId: string, token: string ): Observable { + if (deleted === 99) { + console.log(`reaching rate limit`) + delay(5000) + } + deleted += 1 return from( graphql(token, mutation, { packageVersionId, @@ -51,6 +58,7 @@ export function deletePackageVersions( const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe( tap(result => { + console.log(`versions Deleted 0 : ${deleted}`) if (result) { console.log(`version with id: ${id}, deleted`) } else { diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index dad6909..5d91e6d 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -101,7 +101,7 @@ export function queryForOldestVersions( owner, repo, package: packageName, - last: numVersions, + last: numVersions > 100 ? 100 : numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } From 324039ba0a2f9b3e6099a2d32812699a8f72a51d Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 15:22:21 +0000 Subject: [PATCH 26/45] logs to check rate limit --- dist/index.js | 24 ++++++++++++++++++- src/version/delete-version.ts | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/dist/index.js b/dist/index.js index 52ec2ab..fd4f98d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -177,7 +177,7 @@ exports.Input = Input; "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.deletePackageVersions = exports.deletePackageVersion = void 0; +exports.deletePackageVersions = exports.deletePackageVersion = exports.getRateLimit = void 0; const rxjs_1 = __nccwpck_require__(5805); const operators_1 = __nccwpck_require__(7801); const graphql_1 = __nccwpck_require__(6320); @@ -188,7 +188,29 @@ const mutation = ` success } }`; +const ratelimitQuery = ` +query { + viewer { + login + } + rateLimit { + limit + cost + remaining + resetAt + node + } +}`; +function getRateLimit(token) { + return rxjs_1.from(graphql_1.graphql(token, ratelimitQuery, { + headers: { + Accept: 'application/vnd.github.package-deletes-preview+json' + } + })); +} +exports.getRateLimit = getRateLimit; function deletePackageVersion(packageVersionId, token) { + getRateLimit(token).pipe(operators_1.map(value => console.log(`login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}, node: ${value.ratelimit.node}`))); if (deleted === 99) { console.log(`reaching rate limit`); operators_1.delay(5000); diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 1e6c079..b158815 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -18,10 +18,54 @@ const mutation = ` } }` +export interface RateLimitResponse { + viewer: { + login: string + } + ratelimit: { + limit: number + cost: number + remaining: number + resetAt: string + node: number + } +} + +const ratelimitQuery = ` +query { + viewer { + login + } + rateLimit { + limit + cost + remaining + resetAt + node + } +}` + +export function getRateLimit(token: string): Observable { + return from( + graphql(token, ratelimitQuery, { + headers: { + Accept: 'application/vnd.github.package-deletes-preview+json' + } + }) as Promise + ) +} + export function deletePackageVersion( packageVersionId: string, token: string ): Observable { + getRateLimit(token).pipe( + map(value => + console.log( + `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}, node: ${value.ratelimit.node}` + ) + ) + ) if (deleted === 99) { console.log(`reaching rate limit`) delay(5000) From 372546727d8ab7f408d3db6583dcc103be8b7f59 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 15:30:35 +0000 Subject: [PATCH 27/45] add error handling --- dist/index.js | 4 ++++ src/version/delete-version.ts | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/dist/index.js b/dist/index.js index fd4f98d..61dc595 100644 --- a/dist/index.js +++ b/dist/index.js @@ -206,6 +206,10 @@ function getRateLimit(token) { headers: { Accept: 'application/vnd.github.package-deletes-preview+json' } + })).pipe(operators_1.catchError(err => { + return rxjs_1.throwError(err.errors && err.errors.length > 0 + ? `${err.errors[0].message}` + : `unknown error`); })); } exports.getRateLimit = getRateLimit; diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index b158815..5de131f 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -52,6 +52,14 @@ export function getRateLimit(token: string): Observable { Accept: 'application/vnd.github.package-deletes-preview+json' } }) as Promise + ).pipe( + catchError(err => { + return throwError( + err.errors && err.errors.length > 0 + ? `${err.errors[0].message}` + : `unknown error` + ) + }) ) } From 1369f0ca05c355c68d7a5dc5e33bb54b69ae4a10 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 15:33:41 +0000 Subject: [PATCH 28/45] fix ratelimit query --- dist/index.js | 3 +-- src/version/delete-version.ts | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dist/index.js b/dist/index.js index 61dc595..1b9197b 100644 --- a/dist/index.js +++ b/dist/index.js @@ -198,7 +198,6 @@ query { cost remaining resetAt - node } }`; function getRateLimit(token) { @@ -214,7 +213,7 @@ function getRateLimit(token) { } exports.getRateLimit = getRateLimit; function deletePackageVersion(packageVersionId, token) { - getRateLimit(token).pipe(operators_1.map(value => console.log(`login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}, node: ${value.ratelimit.node}`))); + getRateLimit(token).pipe(operators_1.map(value => console.log(`login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}`))); if (deleted === 99) { console.log(`reaching rate limit`); operators_1.delay(5000); diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 5de131f..73ed706 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -27,7 +27,6 @@ export interface RateLimitResponse { cost: number remaining: number resetAt: string - node: number } } @@ -41,7 +40,6 @@ query { cost remaining resetAt - node } }` @@ -70,7 +68,7 @@ export function deletePackageVersion( getRateLimit(token).pipe( map(value => console.log( - `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}, node: ${value.ratelimit.node}` + `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}` ) ) ) From ef5d9ac3985412ac727c24e77917e803e19d8601 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 15:45:39 +0000 Subject: [PATCH 29/45] add logs --- dist/index.js | 1 + src/version/delete-version.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/dist/index.js b/dist/index.js index 1b9197b..be9b59b 100644 --- a/dist/index.js +++ b/dist/index.js @@ -219,6 +219,7 @@ function deletePackageVersion(packageVersionId, token) { operators_1.delay(5000); } deleted += 1; + operators_1.delay(10000); return rxjs_1.from(graphql_1.graphql(token, mutation, { packageVersionId, headers: { diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 73ed706..f1de9a3 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -77,6 +77,7 @@ export function deletePackageVersion( delay(5000) } deleted += 1 + delay(10000) return from( graphql(token, mutation, { packageVersionId, From 0739cce1e8d24224c6a7e0f4cdbb444db3a37f6f Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 15:49:48 +0000 Subject: [PATCH 30/45] use tap --- __tests__/version/delete-version.test.ts | 10 +++++----- src/version/delete-version.ts | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/__tests__/version/delete-version.test.ts b/__tests__/version/delete-version.test.ts index a34bbbf..44ff9b6 100644 --- a/__tests__/version/delete-version.test.ts +++ b/__tests__/version/delete-version.test.ts @@ -2,10 +2,10 @@ import {deletePackageVersion, deletePackageVersions} from '../../src/version' const githubToken = process.env.GITHUB_TOKEN as string -describe.skip('delete tests', () => { +describe('delete tests', () => { it('deletePackageVersion', async () => { const response = await deletePackageVersion( - 'MDE0OlBhY2thZ2VWZXJzaW9uNjg5OTU1', + 'PV_lADOGReZt84AEI7FzgDSHEI', githubToken ).toPromise() expect(response).toBe(true) @@ -14,9 +14,9 @@ describe.skip('delete tests', () => { it('deletePackageVersions', async () => { const response = await deletePackageVersions( [ - 'MDE0OlBhY2thZ2VWZXJzaW9uNjk4Mjc0', - 'MDE0OlBhY2thZ2VWZXJzaW9uNjk4Mjcx', - 'MDE0OlBhY2thZ2VWZXJzaW9uNjk4MjY3' + 'PV_lADOGReZt84AEI7FzgDSHDs', + 'PV_lADOGReZt84AEI7FzgDSHDY', + 'PV_lADOGReZt84AEI7FzgDSHC8' ], githubToken ).toPromise() diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index f1de9a3..d0f7112 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -66,7 +66,7 @@ export function deletePackageVersion( token: string ): Observable { getRateLimit(token).pipe( - map(value => + tap(value => console.log( `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}` ) From 9ff1e31776ed3647e8a0e6f20b04be79093acb8e Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 16:05:18 +0000 Subject: [PATCH 31/45] use promise for rate limit --- dist/index.js | 29 ++++++++++++++++++----------- src/version/delete-version.ts | 31 ++++++++++--------------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/dist/index.js b/dist/index.js index be9b59b..a94d506 100644 --- a/dist/index.js +++ b/dist/index.js @@ -172,10 +172,19 @@ exports.Input = Input; /***/ }), /***/ 5544: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; Object.defineProperty(exports, "__esModule", ({ value: true })); exports.deletePackageVersions = exports.deletePackageVersion = exports.getRateLimit = void 0; const rxjs_1 = __nccwpck_require__(5805); @@ -201,19 +210,17 @@ query { } }`; function getRateLimit(token) { - return rxjs_1.from(graphql_1.graphql(token, ratelimitQuery, { - headers: { - Accept: 'application/vnd.github.package-deletes-preview+json' - } - })).pipe(operators_1.catchError(err => { - return rxjs_1.throwError(err.errors && err.errors.length > 0 - ? `${err.errors[0].message}` - : `unknown error`); - })); + return __awaiter(this, void 0, void 0, function* () { + return graphql_1.graphql(token, ratelimitQuery, { + headers: { + Accept: 'application/vnd.github.package-deletes-preview+json' + } + }); + }); } exports.getRateLimit = getRateLimit; function deletePackageVersion(packageVersionId, token) { - getRateLimit(token).pipe(operators_1.map(value => console.log(`login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}`))); + getRateLimit(token).then(value => console.log(`login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}`)); if (deleted === 99) { console.log(`reaching rate limit`); operators_1.delay(5000); diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index d0f7112..64f390d 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -43,35 +43,24 @@ query { } }` -export function getRateLimit(token: string): Observable { - return from( - graphql(token, ratelimitQuery, { - headers: { - Accept: 'application/vnd.github.package-deletes-preview+json' - } - }) as Promise - ).pipe( - catchError(err => { - return throwError( - err.errors && err.errors.length > 0 - ? `${err.errors[0].message}` - : `unknown error` - ) - }) - ) +export async function getRateLimit(token: string): Promise { + return graphql(token, ratelimitQuery, { + headers: { + Accept: 'application/vnd.github.package-deletes-preview+json' + } + }) as Promise } export function deletePackageVersion( packageVersionId: string, token: string ): Observable { - getRateLimit(token).pipe( - tap(value => - console.log( - `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}` - ) + getRateLimit(token).then(value => + console.log( + `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}` ) ) + if (deleted === 99) { console.log(`reaching rate limit`) delay(5000) From e9faf3233c9ef3187565121e358fbe57793d0c45 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 16:17:44 +0000 Subject: [PATCH 32/45] check rate limit --- dist/index.js | 8 +++++++- src/version/delete-version.ts | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/dist/index.js b/dist/index.js index a94d506..1a0455d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -220,7 +220,13 @@ function getRateLimit(token) { } exports.getRateLimit = getRateLimit; function deletePackageVersion(packageVersionId, token) { - getRateLimit(token).then(value => console.log(`login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}`)); + getRateLimit(token) + .then(value => console.log(`login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}`)) + .catch(err => { + return rxjs_1.throwError(err.errors && err.errors.length > 0 + ? `${err.errors[0].message}` + : `Unknown Error`); + }); if (deleted === 99) { console.log(`reaching rate limit`); operators_1.delay(5000); diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 64f390d..0f494d9 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -55,11 +55,19 @@ export function deletePackageVersion( packageVersionId: string, token: string ): Observable { - getRateLimit(token).then(value => - console.log( - `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}` + getRateLimit(token) + .then(value => + console.log( + `login: ${value.viewer.login}, rate limit: ${value.ratelimit.limit}, cost: ${value.ratelimit.cost}, remaining: ${value.ratelimit.remaining}` + ) ) - ) + .catch(err => { + return throwError( + err.errors && err.errors.length > 0 + ? `${err.errors[0].message}` + : `Unknown Error` + ) + }) if (deleted === 99) { console.log(`reaching rate limit`) From bf6c02443aea06952db97f1f86a0d9a5ccb96274 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 19:09:58 +0000 Subject: [PATCH 33/45] run with RATE_LIMIT=100 --- dist/index.js | 27 +++++++++++++++++---------- src/delete.ts | 23 ++++++++++++----------- src/version/delete-version.ts | 12 ++++++++++-- src/version/get-versions.ts | 2 +- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/dist/index.js b/dist/index.js index 1eb34df..c133e5c 100644 --- a/dist/index.js +++ b/dist/index.js @@ -27,12 +27,12 @@ exports.deleteVersions = exports.finalIds = exports.getVersionIds = void 0; const rxjs_1 = __nccwpck_require__(5805); const version_1 = __nccwpck_require__(4428); const operators_1 = __nccwpck_require__(7801); -const RATE_LIMIT = 99; -let totalCount; +const RATE_LIMIT = 100; +let totalCount = 0; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate ? version_1.getOldestVersions(owner, repo, packageName, numVersions, value.cursor, token) - : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = value.totalCount)), operators_1.map(value => value.versions)); + : rxjs_1.EMPTY), operators_1.tap(value => (totalCount === 0 ? value.totalCount : totalCount)), operators_1.map(value => value.versions)); } exports.getVersionIds = getVersionIds; function finalIds(input) { @@ -45,17 +45,16 @@ function finalIds(input) { input.numOldVersionsToDelete < RATE_LIMIT ? input.numOldVersionsToDelete : RATE_LIMIT; + console.log(`input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`); return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { + value = value.filter(info => !input.ignoreVersions.test(info.version)); const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length; - input.numDeleted += value.filter(info => !input.ignoreVersions.test(info.version)).length; - return value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, temp); + console.log(`temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}`); + return value.map(info => info.id).slice(0, temp); })); } else { @@ -68,7 +67,7 @@ function finalIds(input) { toDelete = toDelete > value.length ? value.length : toDelete; if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { // using input.numDeleted to keep track of deleted and remaining packages - if (input.numDeleted + toDelete > 99) { + if (input.numDeleted + toDelete > RATE_LIMIT) { toDelete = RATE_LIMIT - input.numDeleted; input.numDeleted = RATE_LIMIT; } @@ -177,6 +176,7 @@ exports.deletePackageVersions = exports.deletePackageVersion = void 0; const rxjs_1 = __nccwpck_require__(5805); const operators_1 = __nccwpck_require__(7801); const graphql_1 = __nccwpck_require__(6320); +let deleted = 0; const mutation = ` mutation deletePackageVersion($packageVersionId: String!) { deletePackageVersion(input: {packageVersionId: $packageVersionId}) { @@ -184,6 +184,11 @@ const mutation = ` } }`; function deletePackageVersion(packageVersionId, token) { + if (deleted === 100) { + console.log(`reaching rate limit`); + operators_1.delay(5000); + } + deleted += 1; return rxjs_1.from(graphql_1.graphql(token, mutation, { packageVersionId, headers: { @@ -202,6 +207,7 @@ function deletePackageVersions(packageVersionIds, token) { return rxjs_1.of(true); } const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { + console.log(`versions Deleted 0 : ${deleted}`); if (result) { console.log(`version with id: ${id}, deleted`); } @@ -209,6 +215,7 @@ function deletePackageVersions(packageVersionIds, token) { console.log(`version with id: ${id}, not deleted`); } }))); + console.log(`Versions Deleted Final2: ${deleted}`); return rxjs_1.merge(...deletes); } exports.deletePackageVersions = deletePackageVersions; @@ -282,7 +289,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs owner, repo, package: packageName, - last: numVersions, + last: numVersions > 100 ? 100 : numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } diff --git a/src/delete.ts b/src/delete.ts index 330e2b6..ef29626 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -3,8 +3,8 @@ import {EMPTY, Observable, of, throwError} from 'rxjs' import {deletePackageVersions, getOldestVersions, VersionInfo} from './version' import {concatMap, map, expand, tap} from 'rxjs/operators' -const RATE_LIMIT = 99 -let totalCount: number +const RATE_LIMIT = 100 +let totalCount = 0 export function getVersionIds( owner: string, @@ -34,7 +34,7 @@ export function getVersionIds( ) : EMPTY ), - tap(value => (totalCount = value.totalCount)), + tap(value => (totalCount === 0 ? value.totalCount : totalCount)), map(value => value.versions) ) } @@ -49,6 +49,9 @@ export function finalIds(input: Input): Observable { input.numOldVersionsToDelete < RATE_LIMIT ? input.numOldVersionsToDelete : RATE_LIMIT + console.log( + `input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}` + ) return getVersionIds( input.owner, input.repo, @@ -58,18 +61,16 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { + value = value.filter(info => !input.ignoreVersions.test(info.version)) const temp = input.numOldVersionsToDelete input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length - input.numDeleted += value.filter( - info => !input.ignoreVersions.test(info.version) - ).length - return value - .filter(info => !input.ignoreVersions.test(info.version)) - .map(info => info.id) - .slice(0, temp) + console.log( + `temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}` + ) + return value.map(info => info.id).slice(0, temp) }) ) } else { @@ -90,7 +91,7 @@ export function finalIds(input: Input): Observable { toDelete = toDelete > value.length ? value.length : toDelete if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { // using input.numDeleted to keep track of deleted and remaining packages - if (input.numDeleted + toDelete > 99) { + if (input.numDeleted + toDelete > RATE_LIMIT) { toDelete = RATE_LIMIT - input.numDeleted input.numDeleted = RATE_LIMIT } else { diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 40a7269..07b20b7 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -1,8 +1,10 @@ import {from, Observable, merge, throwError, of} from 'rxjs' -import {catchError, map, tap} from 'rxjs/operators' +import {catchError, delay, map, tap} from 'rxjs/operators' import {GraphQlQueryResponse} from '@octokit/graphql/dist-types/types' import {graphql} from './graphql' +let deleted = 0 + export interface DeletePackageVersionMutationResponse { deletePackageVersion: { success: boolean @@ -20,6 +22,11 @@ export function deletePackageVersion( packageVersionId: string, token: string ): Observable { + if (deleted === 100) { + console.log(`reaching rate limit`) + delay(5000) + } + deleted += 1 return from( graphql(token, mutation, { packageVersionId, @@ -51,6 +58,7 @@ export function deletePackageVersions( const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe( tap(result => { + console.log(`versions Deleted 0 : ${deleted}`) if (result) { console.log(`version with id: ${id}, deleted`) } else { @@ -59,6 +67,6 @@ export function deletePackageVersions( }) ) ) - + console.log(`Versions Deleted Final2: ${deleted}`) return merge(...deletes) } diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index dad6909..5d91e6d 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -101,7 +101,7 @@ export function queryForOldestVersions( owner, repo, package: packageName, - last: numVersions, + last: numVersions > 100 ? 100 : numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } From 1a7a0291b8a2e8b96794bfb7cc7da2cef0509203 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 19:21:20 +0000 Subject: [PATCH 34/45] RATE_LIMIT=100 --- dist/index.js | 62 +++++++++-------------------------- src/delete.ts | 2 +- src/version/delete-version.ts | 33 ------------------- 3 files changed, 17 insertions(+), 80 deletions(-) diff --git a/dist/index.js b/dist/index.js index 0671e1b..95ac5b9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -58,30 +58,31 @@ function finalIds(input) { })); } else { - return getVersionIds(input.owner, input.repo, input.packageName, 100, '', input.token).pipe(operators_1.map(value => { + return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => { totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length; value = value.filter(info => !input.ignoreVersions.test(info.version)); let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted; toDelete = toDelete > value.length ? value.length : toDelete; - /* if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { - // using input.numDeleted to keep track of deleted and remaining packages - if (input.numDeleted + toDelete > RATE_LIMIT) { - toDelete = RATE_LIMIT - input.numDeleted - input.numDeleted = RATE_LIMIT - } else { - input.numDeleted = input.numDeleted + toDelete - } - return value.map(info => info.id).slice(0, toDelete) - } else return []*/ - if (toDelete > 0) { - input.numDeleted += toDelete; + // using input.numDeleted to keep track of deleted and remaining packages + if (input.numDeleted + toDelete > RATE_LIMIT) { + toDelete = RATE_LIMIT - input.numDeleted; + input.numDeleted = RATE_LIMIT; + } + else { + input.numDeleted = input.numDeleted + toDelete; + } return value.map(info => info.id).slice(0, toDelete); } else return []; + /* + if (toDelete > 0) { + input.numDeleted += toDelete + return value.map(info => info.id).slice(0, toDelete) + } else return []*/ })); } } @@ -171,21 +172,12 @@ exports.Input = Input; /***/ }), /***/ 5544: -/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.deletePackageVersions = exports.deletePackageVersion = exports.getRateLimit = void 0; +exports.deletePackageVersions = exports.deletePackageVersion = void 0; const rxjs_1 = __nccwpck_require__(5805); const operators_1 = __nccwpck_require__(7801); const graphql_1 = __nccwpck_require__(6320); @@ -196,28 +188,6 @@ const mutation = ` success } }`; -const ratelimitQuery = ` -query { - viewer { - login - } - rateLimit { - limit - cost - remaining - resetAt - } -}`; -function getRateLimit(token) { - return __awaiter(this, void 0, void 0, function* () { - return graphql_1.graphql(token, ratelimitQuery, { - headers: { - Accept: 'application/vnd.github.package-deletes-preview+json' - } - }); - }); -} -exports.getRateLimit = getRateLimit; function deletePackageVersion(packageVersionId, token) { if (deleted === 100) { console.log(`reaching rate limit`); diff --git a/src/delete.ts b/src/delete.ts index 403894e..22d2191 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -78,7 +78,7 @@ export function finalIds(input: Input): Observable { input.owner, input.repo, input.packageName, - 100, + RATE_LIMIT, '', input.token ).pipe( diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index e7b7814..07b20b7 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -18,39 +18,6 @@ const mutation = ` } }` -export interface RateLimitResponse { - viewer: { - login: string - } - ratelimit: { - limit: number - cost: number - remaining: number - resetAt: string - } -} - -const ratelimitQuery = ` -query { - viewer { - login - } - rateLimit { - limit - cost - remaining - resetAt - } -}` - -export async function getRateLimit(token: string): Promise { - return graphql(token, ratelimitQuery, { - headers: { - Accept: 'application/vnd.github.package-deletes-preview+json' - } - }) as Promise -} - export function deletePackageVersion( packageVersionId: string, token: string From 8283350471d9e648bf28ae366f80671aa50d528e Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Wed, 22 Dec 2021 21:39:57 +0000 Subject: [PATCH 35/45] fix totalCount --- dist/index.js | 4 +++- src/delete.ts | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index 95ac5b9..4e61ba8 100644 --- a/dist/index.js +++ b/dist/index.js @@ -32,7 +32,7 @@ let totalCount = 0; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate ? version_1.getOldestVersions(owner, repo, packageName, numVersions, value.cursor, token) - : rxjs_1.EMPTY), operators_1.tap(value => (totalCount === 0 ? value.totalCount : totalCount)), operators_1.map(value => value.versions)); + : rxjs_1.EMPTY), operators_1.tap(value => console.log(`total3: ${totalCount}, value total3: ${value.totalCount}`)), operators_1.tap(value => (totalCount = totalCount === 0 ? value.totalCount : totalCount)), operators_1.map(value => value.versions)); } exports.getVersionIds = getVersionIds; function finalIds(input) { @@ -47,6 +47,7 @@ function finalIds(input) { : RATE_LIMIT; console.log(`input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`); return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { + console.log(`totalCount in numVersions: ${totalCount}`); value = value.filter(info => !input.ignoreVersions.test(info.version)); const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = @@ -59,6 +60,7 @@ function finalIds(input) { } else { return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => { + console.log(`total count: ${totalCount}`); totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length; diff --git a/src/delete.ts b/src/delete.ts index 22d2191..9880e78 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -34,7 +34,12 @@ export function getVersionIds( ) : EMPTY ), - tap(value => (totalCount === 0 ? value.totalCount : totalCount)), + tap(value => + console.log(`total3: ${totalCount}, value total3: ${value.totalCount}`) + ), + tap( + value => (totalCount = totalCount === 0 ? value.totalCount : totalCount) + ), map(value => value.versions) ) } @@ -61,6 +66,7 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { + console.log(`totalCount in numVersions: ${totalCount}`) value = value.filter(info => !input.ignoreVersions.test(info.version)) const temp = input.numOldVersionsToDelete input.numOldVersionsToDelete = @@ -83,6 +89,7 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { + console.log(`total count: ${totalCount}`) totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length From f556b07dce45d90d66380d9848d242790157389e Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 23 Dec 2021 08:41:06 +0000 Subject: [PATCH 36/45] check error --- dist/index.js | 2 +- src/version/delete-version.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index 4e61ba8..e686b55 100644 --- a/dist/index.js +++ b/dist/index.js @@ -205,7 +205,7 @@ function deletePackageVersion(packageVersionId, token) { const msg = 'delete version mutation failed.'; return rxjs_1.throwError(err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` - : `${msg} verify input parameters are correct`); + : `${msg} verify input parameters are correct// error123: ${err}`); }), operators_1.map(response => response.deletePackageVersion.success)); } exports.deletePackageVersion = deletePackageVersion; diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 07b20b7..4bf9125 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -40,7 +40,7 @@ export function deletePackageVersion( return throwError( err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` - : `${msg} verify input parameters are correct` + : `${msg} verify input parameters are correct// error123: ${err}` ) }), map(response => response.deletePackageVersion.success) From cfbae0ac37e9c7327c747ceab8802ba72c2b056c Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 23 Dec 2021 09:46:46 +0000 Subject: [PATCH 37/45] remove excess logs --- dist/index.js | 26 +++++--------------------- src/delete.ts | 22 +--------------------- src/version/delete-version.ts | 14 ++++---------- 3 files changed, 10 insertions(+), 52 deletions(-) diff --git a/dist/index.js b/dist/index.js index e686b55..766fd6b 100644 --- a/dist/index.js +++ b/dist/index.js @@ -32,7 +32,7 @@ let totalCount = 0; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate ? version_1.getOldestVersions(owner, repo, packageName, numVersions, value.cursor, token) - : rxjs_1.EMPTY), operators_1.tap(value => console.log(`total3: ${totalCount}, value total3: ${value.totalCount}`)), operators_1.tap(value => (totalCount = totalCount === 0 ? value.totalCount : totalCount)), operators_1.map(value => value.versions)); + : rxjs_1.EMPTY), operators_1.tap(value => (totalCount = totalCount === 0 ? value.totalCount : totalCount)), operators_1.map(value => value.versions)); } exports.getVersionIds = getVersionIds; function finalIds(input) { @@ -47,20 +47,17 @@ function finalIds(input) { : RATE_LIMIT; console.log(`input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`); return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { - console.log(`totalCount in numVersions: ${totalCount}`); value = value.filter(info => !input.ignoreVersions.test(info.version)); const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length; - console.log(`temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}`); return value.map(info => info.id).slice(0, temp); })); } else { return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => { - console.log(`total count: ${totalCount}`); totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length; @@ -80,11 +77,6 @@ function finalIds(input) { } else return []; - /* - if (toDelete > 0) { - input.numDeleted += toDelete - return value.map(info => info.id).slice(0, toDelete) - } else return []*/ })); } } @@ -103,10 +95,7 @@ function deleteVersions(input) { return rxjs_1.of(true); } const result = finalIds(input); - return result.pipe(operators_1.tap(() => { - if (input.numDeleted > 0) - console.log(`${input.numDeleted} versions will be deleted`); - }), operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); + return result.pipe(operators_1.concatMap(ids => version_1.deletePackageVersions(ids, input.token))); } exports.deleteVersions = deleteVersions; @@ -191,21 +180,17 @@ const mutation = ` } }`; function deletePackageVersion(packageVersionId, token) { - if (deleted === 100) { - console.log(`reaching rate limit`); - operators_1.delay(5000); - } deleted += 1; return rxjs_1.from(graphql_1.graphql(token, mutation, { packageVersionId, headers: { Accept: 'application/vnd.github.package-deletes-preview+json' } - })).pipe(operators_1.catchError((err) => { + })).pipe(operators_1.catchError(err => { const msg = 'delete version mutation failed.'; return rxjs_1.throwError(err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` - : `${msg} verify input parameters are correct// error123: ${err}`); + : `${msg} ${err.message}`); }), operators_1.map(response => response.deletePackageVersion.success)); } exports.deletePackageVersion = deletePackageVersion; @@ -214,7 +199,6 @@ function deletePackageVersions(packageVersionIds, token) { return rxjs_1.of(true); } const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { - console.log(`versions Deleted 0 : ${deleted}`); if (result) { console.log(`version with id: ${id}, deleted`); } @@ -222,7 +206,7 @@ function deletePackageVersions(packageVersionIds, token) { console.log(`version with id: ${id}, not deleted`); } }))); - console.log(`Versions Deleted Final2: ${deleted}`); + console.log(`Versions Deleted: ${deleted}`); return rxjs_1.merge(...deletes); } exports.deletePackageVersions = deletePackageVersions; diff --git a/src/delete.ts b/src/delete.ts index 9880e78..a3d53c4 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -34,9 +34,6 @@ export function getVersionIds( ) : EMPTY ), - tap(value => - console.log(`total3: ${totalCount}, value total3: ${value.totalCount}`) - ), tap( value => (totalCount = totalCount === 0 ? value.totalCount : totalCount) ), @@ -66,16 +63,12 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { - console.log(`totalCount in numVersions: ${totalCount}`) value = value.filter(info => !input.ignoreVersions.test(info.version)) const temp = input.numOldVersionsToDelete input.numOldVersionsToDelete = input.numOldVersionsToDelete - value.length <= 0 ? 0 : input.numOldVersionsToDelete - value.length - console.log( - `temp: ${temp}, numOldeVersions: ${input.numOldVersionsToDelete}` - ) return value.map(info => info.id).slice(0, temp) }) ) @@ -89,7 +82,6 @@ export function finalIds(input: Input): Observable { input.token ).pipe( map(value => { - console.log(`total count: ${totalCount}`) totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length @@ -106,16 +98,10 @@ export function finalIds(input: Input): Observable { } return value.map(info => info.id).slice(0, toDelete) } else return [] - /* - if (toDelete > 0) { - input.numDeleted += toDelete - return value.map(info => info.id).slice(0, toDelete) - } else return []*/ }) ) } } - return throwError( "Could not get packageVersionIds. Explicitly specify using the 'package-version-ids' input" ) @@ -139,11 +125,5 @@ export function deleteVersions(input: Input): Observable { const result = finalIds(input) - return result.pipe( - tap(() => { - if (input.numDeleted > 0) - console.log(`${input.numDeleted} versions will be deleted`) - }), - concatMap(ids => deletePackageVersions(ids, input.token)) - ) + return result.pipe(concatMap(ids => deletePackageVersions(ids, input.token))) } diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index 4bf9125..ef1996d 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -1,6 +1,5 @@ import {from, Observable, merge, throwError, of} from 'rxjs' -import {catchError, delay, map, tap} from 'rxjs/operators' -import {GraphQlQueryResponse} from '@octokit/graphql/dist-types/types' +import {catchError, map, tap} from 'rxjs/operators' import {graphql} from './graphql' let deleted = 0 @@ -22,10 +21,6 @@ export function deletePackageVersion( packageVersionId: string, token: string ): Observable { - if (deleted === 100) { - console.log(`reaching rate limit`) - delay(5000) - } deleted += 1 return from( graphql(token, mutation, { @@ -35,12 +30,12 @@ export function deletePackageVersion( } }) as Promise ).pipe( - catchError((err: GraphQlQueryResponse) => { + catchError(err => { const msg = 'delete version mutation failed.' return throwError( err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` - : `${msg} verify input parameters are correct// error123: ${err}` + : `${msg} ${err.message}` ) }), map(response => response.deletePackageVersion.success) @@ -58,7 +53,6 @@ export function deletePackageVersions( const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe( tap(result => { - console.log(`versions Deleted 0 : ${deleted}`) if (result) { console.log(`version with id: ${id}, deleted`) } else { @@ -67,6 +61,6 @@ export function deletePackageVersions( }) ) ) - console.log(`Versions Deleted Final2: ${deleted}`) + console.log(`Versions Deleted: ${deleted}`) return merge(...deletes) } From 81676df3384e65bfb05a59a5987a0dc5a863a236 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 23 Dec 2021 13:01:35 +0000 Subject: [PATCH 38/45] add comments --- dist/index.js | 46 ++++++++++++++++++++++++++--------- src/delete.ts | 31 +++++++++++++++++++---- src/version/delete-version.ts | 8 +++--- src/version/get-versions.ts | 4 +-- 4 files changed, 65 insertions(+), 24 deletions(-) diff --git a/dist/index.js b/dist/index.js index 766fd6b..9f0b1cb 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41,12 +41,19 @@ function finalIds(input) { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + // This code block is when num-old-versions-to-delete is specified. + // Setting input.numOldVersionsToDelete is set as minimum of input.numOldVersionsToDelete and RATE_LIMIT input.numOldVersionsToDelete = input.numOldVersionsToDelete < RATE_LIMIT ? input.numOldVersionsToDelete : RATE_LIMIT; - console.log(`input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}`); - return getVersionIds(input.owner, input.repo, input.packageName, input.numOldVersionsToDelete, '', input.token).pipe(operators_1.map(value => { + return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe( + // This code block executes on batches of 100 versions starting from oldest + operators_1.map(value => { + /* + Here first filter out the versions that are to be ignored. + Then update input.numOldeVersionsToDelete to the no of versions deleted from the next 100 versions batch. + */ value = value.filter(info => !input.ignoreVersions.test(info.version)); const temp = input.numOldVersionsToDelete; input.numOldVersionsToDelete = @@ -57,15 +64,33 @@ function finalIds(input) { })); } else { - return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe(operators_1.map(value => { + // This code block is when min-versions-to-keep is specified. + return getVersionIds(input.owner, input.repo, input.packageName, RATE_LIMIT, '', input.token).pipe( + // This code block executes on batches of 100 versions starting from oldest + operators_1.map(value => { + /* + Here totalCount is the total no of versions in the package. + First we update totalCount by removing no of ignored versions from it and also filter them out from value. + toDelete is the no of versions that need to be deleted and input.numDeleted is the total no of versions deleted before this batch. + We calculate this from total no of versions in the package, the min no of versions to keep and the no of versions we have deleted in earlier batch. + Then we update toDelete to not exceed the length of current batch of versions. + Now toDelete holds the no of versions to be deleted from the current batch of versions. + */ totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length; value = value.filter(info => !input.ignoreVersions.test(info.version)); let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted; toDelete = toDelete > value.length ? value.length : toDelete; + //Checking here if we have any versions to delete and whether we are within the RATE_LIMIT. if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { - // using input.numDeleted to keep track of deleted and remaining packages + /* + Checking here if we can delete all the versions left in the current batch. + input.numDeleted + toDelete should not exceed RATE_LIMIT. + If it is exceeding we only delete the no of versions from this batch that are allowed within the RATE_LIMIT. + i.e. diff between RATE_LIMIT and versions deleted till now (input.numDeleted) + input.numDeleted is updated accordingly. + */ if (input.numDeleted + toDelete > RATE_LIMIT) { toDelete = RATE_LIMIT - input.numDeleted; input.numDeleted = RATE_LIMIT; @@ -190,7 +215,7 @@ function deletePackageVersion(packageVersionId, token) { const msg = 'delete version mutation failed.'; return rxjs_1.throwError(err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` - : `${msg} ${err.message}`); + : `${msg} ${err.message} \n${deleted - 1} versions deleted till now.`); }), operators_1.map(response => response.deletePackageVersion.success)); } exports.deletePackageVersion = deletePackageVersion; @@ -199,14 +224,11 @@ function deletePackageVersions(packageVersionIds, token) { return rxjs_1.of(true); } const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe(operators_1.tap(result => { - if (result) { - console.log(`version with id: ${id}, deleted`); - } - else { + if (!result) { console.log(`version with id: ${id}, not deleted`); } }))); - console.log(`Versions Deleted: ${deleted}`); + console.log(`Total versions deleted till now: ${deleted}`); return rxjs_1.merge(...deletes); } exports.deletePackageVersions = deletePackageVersions; @@ -280,7 +302,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs owner, repo, package: packageName, - last: numVersions > 100 ? 100 : numVersions, + last: numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } @@ -297,7 +319,7 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs owner, repo, package: packageName, - last: numVersions > 100 ? 100 : numVersions, + last: numVersions, before: startCursor, headers: { Accept: 'application/vnd.github.packages-preview+json' diff --git a/src/delete.ts b/src/delete.ts index a3d53c4..87a4349 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -47,22 +47,26 @@ export function finalIds(input: Input): Observable { } if (input.hasOldestVersionQueryInfo()) { if (input.minVersionsToKeep < 0) { + // This code block is when num-old-versions-to-delete is specified. + // Setting input.numOldVersionsToDelete is set as minimum of input.numOldVersionsToDelete and RATE_LIMIT input.numOldVersionsToDelete = input.numOldVersionsToDelete < RATE_LIMIT ? input.numOldVersionsToDelete : RATE_LIMIT - console.log( - `input.numOldVersionsToDelete: ${input.numOldVersionsToDelete}` - ) return getVersionIds( input.owner, input.repo, input.packageName, - input.numOldVersionsToDelete, + RATE_LIMIT, '', input.token ).pipe( + // This code block executes on batches of 100 versions starting from oldest map(value => { + /* + Here first filter out the versions that are to be ignored. + Then update input.numOldeVersionsToDelete to the no of versions deleted from the next 100 versions batch. + */ value = value.filter(info => !input.ignoreVersions.test(info.version)) const temp = input.numOldVersionsToDelete input.numOldVersionsToDelete = @@ -73,6 +77,7 @@ export function finalIds(input: Input): Observable { }) ) } else { + // This code block is when min-versions-to-keep is specified. return getVersionIds( input.owner, input.repo, @@ -81,15 +86,31 @@ export function finalIds(input: Input): Observable { '', input.token ).pipe( + // This code block executes on batches of 100 versions starting from oldest map(value => { + /* + Here totalCount is the total no of versions in the package. + First we update totalCount by removing no of ignored versions from it and also filter them out from value. + toDelete is the no of versions that need to be deleted and input.numDeleted is the total no of versions deleted before this batch. + We calculate this from total no of versions in the package, the min no of versions to keep and the no of versions we have deleted in earlier batch. + Then we update toDelete to not exceed the length of current batch of versions. + Now toDelete holds the no of versions to be deleted from the current batch of versions. + */ totalCount = totalCount - value.filter(info => input.ignoreVersions.test(info.version)).length value = value.filter(info => !input.ignoreVersions.test(info.version)) let toDelete = totalCount - input.minVersionsToKeep - input.numDeleted toDelete = toDelete > value.length ? value.length : toDelete + //Checking here if we have any versions to delete and whether we are within the RATE_LIMIT. if (toDelete > 0 && input.numDeleted < RATE_LIMIT) { - // using input.numDeleted to keep track of deleted and remaining packages + /* + Checking here if we can delete all the versions left in the current batch. + input.numDeleted + toDelete should not exceed RATE_LIMIT. + If it is exceeding we only delete the no of versions from this batch that are allowed within the RATE_LIMIT. + i.e. diff between RATE_LIMIT and versions deleted till now (input.numDeleted) + input.numDeleted is updated accordingly. + */ if (input.numDeleted + toDelete > RATE_LIMIT) { toDelete = RATE_LIMIT - input.numDeleted input.numDeleted = RATE_LIMIT diff --git a/src/version/delete-version.ts b/src/version/delete-version.ts index ef1996d..734f285 100644 --- a/src/version/delete-version.ts +++ b/src/version/delete-version.ts @@ -35,7 +35,7 @@ export function deletePackageVersion( return throwError( err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` - : `${msg} ${err.message}` + : `${msg} ${err.message} \n${deleted - 1} versions deleted till now.` ) }), map(response => response.deletePackageVersion.success) @@ -53,14 +53,12 @@ export function deletePackageVersions( const deletes = packageVersionIds.map(id => deletePackageVersion(id, token).pipe( tap(result => { - if (result) { - console.log(`version with id: ${id}, deleted`) - } else { + if (!result) { console.log(`version with id: ${id}, not deleted`) } }) ) ) - console.log(`Versions Deleted: ${deleted}`) + console.log(`Total versions deleted till now: ${deleted}`) return merge(...deletes) } diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index 5d91e6d..ee72d03 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -101,7 +101,7 @@ export function queryForOldestVersions( owner, repo, package: packageName, - last: numVersions > 100 ? 100 : numVersions, + last: numVersions, headers: { Accept: 'application/vnd.github.packages-preview+json' } @@ -123,7 +123,7 @@ export function queryForOldestVersions( owner, repo, package: packageName, - last: numVersions > 100 ? 100 : numVersions, + last: numVersions, before: startCursor, headers: { Accept: 'application/vnd.github.packages-preview+json' From 7d188d17769c82061f17b759934daecb19d2500e Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 23 Dec 2021 13:02:25 +0000 Subject: [PATCH 39/45] RATE_LIMIT = 99 --- dist/index.js | 2 +- src/delete.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index 9f0b1cb..ac3d30a 100644 --- a/dist/index.js +++ b/dist/index.js @@ -27,7 +27,7 @@ exports.deleteVersions = exports.finalIds = exports.getVersionIds = void 0; const rxjs_1 = __nccwpck_require__(5805); const version_1 = __nccwpck_require__(4428); const operators_1 = __nccwpck_require__(7801); -const RATE_LIMIT = 100; +const RATE_LIMIT = 99; let totalCount = 0; function getVersionIds(owner, repo, packageName, numVersions, cursor, token) { return version_1.getOldestVersions(owner, repo, packageName, numVersions, cursor, token).pipe(operators_1.expand(value => value.paginate diff --git a/src/delete.ts b/src/delete.ts index 87a4349..59d12fd 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -3,7 +3,7 @@ import {EMPTY, Observable, of, throwError} from 'rxjs' import {deletePackageVersions, getOldestVersions, VersionInfo} from './version' import {concatMap, map, expand, tap} from 'rxjs/operators' -const RATE_LIMIT = 100 +const RATE_LIMIT = 99 let totalCount = 0 export function getVersionIds( From 72cc4b27b0f09ba28565bda1ebc167746c092c42 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 23 Dec 2021 17:41:07 +0000 Subject: [PATCH 40/45] update tests --- __tests__/delete.test.ts | 42 ++++++++++++++---------- __tests__/version/delete-version.test.ts | 8 ++--- __tests__/version/get-version.test.ts | 12 ++----- dist/index.js | 1 - package.json | 5 +-- src/version/get-versions.ts | 1 - 6 files changed, 34 insertions(+), 35 deletions(-) diff --git a/__tests__/delete.test.ts b/__tests__/delete.test.ts index a8208c7..e3f5e0f 100644 --- a/__tests__/delete.test.ts +++ b/__tests__/delete.test.ts @@ -1,26 +1,34 @@ import {Input, InputParams} from '../src/input' import {deleteVersions, finalIds} from '../src/delete' +jest.setTimeout(10000) describe('index tests -- call graphql', () => { - it('getVersionIds test -- get oldest version', done => { + it('finalIds test -- get oldest version', done => { const numVersions = 1 finalIds(getInput({numOldVersionsToDelete: numVersions})).subscribe(ids => { - expect(ids.length).toBeGreaterThanOrEqual(numVersions) + expect(ids.length).toBe(numVersions) done() }) }) - it('getVersionIds test -- get oldest 3 versions', done => { + it.skip('finalIds test -- get oldest 3 versions', done => { const numVersions = 3 - finalIds(getInput({numOldVersionsToDelete: numVersions})).subscribe(ids => { - expect(ids.length).toBeGreaterThanOrEqual(numVersions) + expect(ids.length).toBe(numVersions) done() }) }) - it('getVersionIds test -- supplied package version id', done => { + it.skip('finalIds test -- get oldest 110 versions', done => { + const numVersions = 110 + + finalIds(getInput({numOldVersionsToDelete: numVersions})).subscribe(ids => { + expect(ids.length).toBe(99), async () => done() + }) + }) + + it('finalIds test -- supplied package version id', done => { const suppliedIds = [ 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', 'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB', @@ -56,26 +64,26 @@ describe('index tests -- call graphql', () => { it.skip('deleteVersions test -- delete oldest version', done => { deleteVersions(getInput({numOldVersionsToDelete: 1})).subscribe( isSuccess => { - expect(isSuccess).toBe(true) - done() - } + expect(isSuccess) + }, + async () => done() ) }) it.skip('deleteVersions test -- delete 3 oldest versions', done => { deleteVersions(getInput({numOldVersionsToDelete: 3})).subscribe( isSuccess => { - expect(isSuccess).toBe(true) - done() - } + expect(isSuccess) + }, + async () => done() ) }) it.skip('deleteVersions test -- keep 5 versions', done => { - deleteVersions(getInput({minVersionsToKeep: 5})).subscribe(isSuccess => { + deleteVersions(getInput({minVersionsToKeep: 100})).subscribe(isSuccess => { expect(isSuccess).toBe(true) - done() - }) + }), + async () => done() }) }) @@ -83,9 +91,9 @@ const defaultInput: InputParams = { packageVersionIds: [], owner: 'namratajha', repo: 'only-pkg', - packageName: 'onlypkg.maven', + packageName: 'only-pkg', numOldVersionsToDelete: 1, - minVersionsToKeep: 1, + minVersionsToKeep: -1, ignoreVersions: RegExp('^$'), token: process.env.GITHUB_TOKEN as string } diff --git a/__tests__/version/delete-version.test.ts b/__tests__/version/delete-version.test.ts index a34bbbf..e93b486 100644 --- a/__tests__/version/delete-version.test.ts +++ b/__tests__/version/delete-version.test.ts @@ -5,7 +5,7 @@ const githubToken = process.env.GITHUB_TOKEN as string describe.skip('delete tests', () => { it('deletePackageVersion', async () => { const response = await deletePackageVersion( - 'MDE0OlBhY2thZ2VWZXJzaW9uNjg5OTU1', + 'PV_lADOGReZt84AEI7FzgDSHEI', githubToken ).toPromise() expect(response).toBe(true) @@ -14,9 +14,9 @@ describe.skip('delete tests', () => { it('deletePackageVersions', async () => { const response = await deletePackageVersions( [ - 'MDE0OlBhY2thZ2VWZXJzaW9uNjk4Mjc0', - 'MDE0OlBhY2thZ2VWZXJzaW9uNjk4Mjcx', - 'MDE0OlBhY2thZ2VWZXJzaW9uNjk4MjY3' + 'PV_lADOGReZt84AEI7FzgDSHDs', + 'PV_lADOGReZt84AEI7FzgDSHDY', + 'PV_lADOGReZt84AEI7FzgDSHC8' ], githubToken ).toPromise() diff --git a/__tests__/version/get-version.test.ts b/__tests__/version/get-version.test.ts index d3a2d72..42a94f2 100644 --- a/__tests__/version/get-version.test.ts +++ b/__tests__/version/get-version.test.ts @@ -14,14 +14,6 @@ describe('get versions tests -- call graphql', () => { }) }) - it('getOldestVersions -- succeeds for more than 100 versions', done => { - const numVersions = 110 - getOldestVersions({numVersions}).subscribe(result => { - expect(result.versions.length).toBe(100) - done() - }) - }) - it('getOldestVersions -- fails for invalid repo', done => { getOldestVersions({repo: 'actions-testin'}).subscribe({ error: err => { @@ -56,8 +48,8 @@ interface Params { const defaultParams = { owner: 'namratajha', - repo: 'only-pkg', - packageName: 'only-pkg', + repo: 'test-repo', + packageName: 'test-repo', numVersions: 1, startCursor: '', token: process.env.GITHUB_TOKEN as string diff --git a/dist/index.js b/dist/index.js index ac3d30a..ae63fbe 100644 --- a/dist/index.js +++ b/dist/index.js @@ -308,7 +308,6 @@ function queryForOldestVersions(owner, repo, packageName, numVersions, startCurs } })).pipe(operators_1.catchError((err) => { const msg = 'query for oldest version failed.'; - console.log(`numversions: ${numVersions} startCursor: ${startCursor}`); return rxjs_1.throwError(err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` : `${msg} verify input parameters are correct`); diff --git a/package.json b/package.json index ca8b82d..b27e031 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,10 @@ "scripts": { "format": "prettier --write **/*.ts", "format-check": "prettier --check **/*.ts", - "lint": "eslint src/**/*.ts", + "lint": "eslint src/**/*.ts --fix", + "lint-check": "eslint src/**/*.ts", "test": "jest", - "build": "npm run format-check && npm run lint && tsc", + "build": "npm run format-check && npm run lint-check && npm run test && tsc", "pack": "rm -rf ./lib ./dist && npm run build && ncc build" }, "repository": { diff --git a/src/version/get-versions.ts b/src/version/get-versions.ts index ee72d03..c285de4 100644 --- a/src/version/get-versions.ts +++ b/src/version/get-versions.ts @@ -109,7 +109,6 @@ export function queryForOldestVersions( ).pipe( catchError((err: GraphQlQueryResponse) => { const msg = 'query for oldest version failed.' - console.log(`numversions: ${numVersions} startCursor: ${startCursor}`) return throwError( err.errors && err.errors.length > 0 ? `${msg} ${err.errors[0].message}` From 406ce0281db0c42aee6604936d7a60c31443f927 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Fri, 24 Dec 2021 00:20:21 +0530 Subject: [PATCH 41/45] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa4bef3..cc54f70 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Delete Package Versions -This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages). This action will only delete a maximum of 100 versions in one run. +This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages). This action will only delete a maximum of 99 versions in one run. To delete more than 99 versions the action can be called multiple times in a workflow. ### What It Can Do @@ -42,7 +42,7 @@ This action deletes versions of a package from [GitHub Packages](https://github. # Defaults to 1. num-old-versions-to-delete: - # The number of latest versions to not delete. + # The number of latest versions to keep. # This cannot be specified with `num-old-versions-to-delete`. By default, `num-old-versions-to-delete` takes precedence over `min-versions-to-keep`. # When set to 0, all deletable versions will be deleted. # When set greater than 0, all deletable package versions except the specified number will be deleted. From aac34a9f3354ee3c26e304066740d686bee4c5f5 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Thu, 23 Dec 2021 18:53:16 +0000 Subject: [PATCH 42/45] update tests --- __tests__/delete.test.ts | 3 +-- __tests__/version/get-version.test.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/__tests__/delete.test.ts b/__tests__/delete.test.ts index e3f5e0f..7dd83c7 100644 --- a/__tests__/delete.test.ts +++ b/__tests__/delete.test.ts @@ -1,8 +1,7 @@ import {Input, InputParams} from '../src/input' import {deleteVersions, finalIds} from '../src/delete' -jest.setTimeout(10000) -describe('index tests -- call graphql', () => { +describe.skip('index tests -- call graphql', () => { it('finalIds test -- get oldest version', done => { const numVersions = 1 diff --git a/__tests__/version/get-version.test.ts b/__tests__/version/get-version.test.ts index 42a94f2..d94de3f 100644 --- a/__tests__/version/get-version.test.ts +++ b/__tests__/version/get-version.test.ts @@ -5,7 +5,7 @@ import { } from '../../src/version' import {Observable} from 'rxjs' -describe('get versions tests -- call graphql', () => { +describe.skip('get versions tests -- call graphql', () => { it('getOldestVersions -- succeeds', done => { const numVersions = 1 getOldestVersions({numVersions}).subscribe(result => { From 5282013b71c4d6bb7dc0394df5657aae7eb151d6 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Fri, 24 Dec 2021 12:35:55 +0530 Subject: [PATCH 43/45] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc54f70..9756d00 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Delete Package Versions -This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages). This action will only delete a maximum of 99 versions in one run. To delete more than 99 versions the action can be called multiple times in a workflow. +This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages). This action will only delete a maximum of 99 versions in one run. ### What It Can Do From f804442fe438b05d2785df75602ddfdd582377b0 Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Fri, 24 Dec 2021 14:10:14 +0530 Subject: [PATCH 44/45] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9756d00..4fd4504 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Delete Package Versions -This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages). This action will only delete a maximum of 99 versions in one run. +This action deletes versions of a package from [GitHub Packages](https://github.com/features/packages) except ghcr packages. This action will only delete a maximum of 99 versions in one run. ### What It Can Do From 6a5e08c48d46d1e49236909a40f2d107f8d8a03f Mon Sep 17 00:00:00 2001 From: Namrata Jha Date: Fri, 24 Dec 2021 10:21:02 +0000 Subject: [PATCH 45/45] fix typo --- dist/index.js | 2 +- src/delete.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/index.js b/dist/index.js index ae63fbe..14c871c 100644 --- a/dist/index.js +++ b/dist/index.js @@ -113,7 +113,7 @@ function deleteVersions(input) { return rxjs_1.throwError('No token found'); } if (!input.checkInput()) { - return rxjs_1.throwError('Invlaid input combination'); + return rxjs_1.throwError('Invalid input combination'); } if (input.numOldVersionsToDelete <= 0 && input.minVersionsToKeep < 0) { console.log('Number of old versions to delete input is 0 or less, no versions will be deleted'); diff --git a/src/delete.ts b/src/delete.ts index 59d12fd..c2f9c3c 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -134,7 +134,7 @@ export function deleteVersions(input: Input): Observable { } if (!input.checkInput()) { - return throwError('Invlaid input combination') + return throwError('Invalid input combination') } if (input.numOldVersionsToDelete <= 0 && input.minVersionsToKeep < 0) {