diff --git a/README.md b/README.md index 1c38a5b00..dca64322e 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ For every single NPM package, we create a record in the Algolia index. The resul deprecated: 'Deprecated', // This field will be removed, please use `isDeprecated` instead isDeprecated: true, deprecatedReason: 'Deprecated', + isSecurityHeld: false, // See https://github.com/npm/security-holder badPackage: false, homepage: 'https://babeljs.io/', license: 'MIT', diff --git a/src/@types/pkg.ts b/src/@types/pkg.ts index 4e802a237..a35efad5a 100644 --- a/src/@types/pkg.ts +++ b/src/@types/pkg.ts @@ -16,6 +16,7 @@ export interface Repo { head?: string; branch?: string; } + export interface GithubRepo { user: string; project: string; @@ -48,6 +49,7 @@ export interface RawPkg { downloadsLast30Days: number; downloadsRatio: number; humanDownloadsLast30Days: string; + jsDelivrHits: number; popular: boolean; version: string; versions: Record; @@ -64,6 +66,7 @@ export interface RawPkg { deprecated: boolean | string; isDeprecated: boolean; deprecatedReason: string | null; + isSecurityHeld: boolean; homepage: string | null; license: string | null; keywords: string[]; @@ -74,13 +77,17 @@ export interface RawPkg { lastPublisher: Owner | null; owners: Owner[]; bin: Record; + dependents: number; types: TsType; moduleTypes: ModuleType[]; styleTypes: StyleType[]; + humanDependents: string; + changelogFilename: string | null; lastCrawl: string; _searchInternal: { expiresAt: number; alternativeNames: string[]; + popularAlternativeNames: string[]; }; } diff --git a/src/__tests__/__snapshots__/formatPkg.test.ts.snap b/src/__tests__/__snapshots__/formatPkg.test.ts.snap index 7ccdac291..16b60ec2d 100644 --- a/src/__tests__/__snapshots__/formatPkg.test.ts.snap +++ b/src/__tests__/__snapshots__/formatPkg.test.ts.snap @@ -9,12 +9,15 @@ Object { "0js", ], "expiresAt": Any, + "popularAlternativeNames": Array [], }, "bin": Object {}, + "changelogFilename": null, "computedKeywords": Array [], "computedMetadata": Object {}, "created": NaN, "dependencies": Object {}, + "dependents": 0, "deprecated": "Yes this is deprecated", "deprecatedReason": "Yes this is deprecated", "description": null, @@ -29,8 +32,11 @@ Object { "user": "algolia", }, "homepage": null, + "humanDependents": "0", "humanDownloadsLast30Days": "0", "isDeprecated": true, + "isSecurityHeld": false, + "jsDelivrHits": 0, "keywords": Array [], "lastCrawl": Any, "lastPublisher": null, @@ -86,8 +92,10 @@ Object { "@atlaskit/input", ], "expiresAt": Any, + "popularAlternativeNames": Array [], }, "bin": Object {}, + "changelogFilename": null, "computedKeywords": Array [], "computedMetadata": Object {}, "created": 1485733638021, @@ -98,6 +106,7 @@ Object { "prop-types": "^15.5.10", "styled-components": "^1.4.6", }, + "dependents": 0, "deprecated": false, "deprecatedReason": null, "description": "Internal component used by field-base to create field-text. DO NOT USE DIRECTLY", @@ -109,8 +118,11 @@ Object { "gitHead": null, "githubRepo": null, "homepage": "https://bitbucket.org/atlassian/atlaskit#readme", + "humanDependents": "0", "humanDownloadsLast30Days": "0", "isDeprecated": false, + "isSecurityHeld": false, + "jsDelivrHits": 0, "keywords": Array [ "atlaskit", "ui", @@ -288,8 +300,10 @@ Object { "@atomic-package/tab", ], "expiresAt": Any, + "popularAlternativeNames": Array [], }, "bin": Object {}, + "changelogFilename": null, "computedKeywords": Array [], "computedMetadata": Object {}, "created": 1498148375102, @@ -297,6 +311,7 @@ Object { "@atomic-package/common": "0.0.1", "@atomic-package/utility": "0.0.2", }, + "dependents": 0, "deprecated": false, "deprecatedReason": null, "description": "atomic-package - tab", @@ -335,8 +350,11 @@ Object { "user": "atomic-package", }, "homepage": null, + "humanDependents": "0", "humanDownloadsLast30Days": "0", "isDeprecated": false, + "isSecurityHeld": false, + "jsDelivrHits": 0, "keywords": Array [], "lastCrawl": Any, "lastPublisher": Object { @@ -412,10 +430,12 @@ Object { "create-instantsearch-app", ], "expiresAt": Any, + "popularAlternativeNames": Array [], }, "bin": Object { "create-instantsearch-app": "src/cli/index.js", }, + "changelogFilename": null, "computedKeywords": Array [], "computedMetadata": Object {}, "created": 1500453698313, @@ -435,6 +455,7 @@ Object { "semver": "6.0.0", "validate-npm-package-name": "3.0.0", }, + "dependents": 0, "deprecated": false, "deprecatedReason": null, "description": "⚡️ Build InstantSearch apps at the speed of thought", @@ -465,8 +486,11 @@ Object { "user": "algolia", }, "homepage": null, + "humanDependents": "0", "humanDownloadsLast30Days": "0", "isDeprecated": false, + "isSecurityHeld": false, + "jsDelivrHits": 0, "keywords": Array [ "algolia", "instantsearch", @@ -540,12 +564,15 @@ Object { "indexofjs", ], "expiresAt": Any, + "popularAlternativeNames": Array [], }, "bin": Object {}, + "changelogFilename": null, "computedKeywords": Array [], "computedMetadata": Object {}, "created": 1346430013243, "dependencies": Object {}, + "dependents": 0, "deprecated": false, "deprecatedReason": null, "description": "Microsoft sucks", @@ -555,8 +582,11 @@ Object { "gitHead": null, "githubRepo": null, "homepage": null, + "humanDependents": "0", "humanDownloadsLast30Days": "0", "isDeprecated": false, + "isSecurityHeld": false, + "jsDelivrHits": 0, "keywords": Array [ "index", "array", @@ -628,14 +658,17 @@ Object { "prism", ], "expiresAt": Any, + "popularAlternativeNames": Array [], }, "bin": Object {}, + "changelogFilename": null, "computedKeywords": Array [], "computedMetadata": Object {}, "created": 1431474880643, "dependencies": Object { "clipboard": "^2.0.0", }, + "dependents": 0, "deprecated": false, "deprecatedReason": null, "description": "Lightweight, robust, elegant syntax highlighting. A spin-off project from Dabblet.", @@ -665,8 +698,11 @@ Object { "user": "LeaVerou", }, "homepage": null, + "humanDependents": "0", "humanDownloadsLast30Days": "0", "isDeprecated": false, + "isSecurityHeld": false, + "jsDelivrHits": 0, "keywords": Array [ "prism", "highlight", @@ -836,12 +872,15 @@ Object { "long-boy", ], "expiresAt": Any, + "popularAlternativeNames": Array [], }, "bin": Object {}, + "changelogFilename": null, "computedKeywords": Array [], "computedMetadata": Object {}, "created": NaN, "dependencies": Object {}, + "dependents": 0, "deprecated": false, "deprecatedReason": null, "description": null, @@ -856,8 +895,11 @@ Object { "user": "algolia", }, "homepage": null, + "humanDependents": "0", "humanDownloadsLast30Days": "0", "isDeprecated": false, + "isSecurityHeld": false, + "jsDelivrHits": 0, "keywords": Array [], "lastCrawl": Any, "lastPublisher": null, @@ -1537,3 +1579,159 @@ QE+S ], } `; + +exports[`security held log security held flag 1`] = ` +Object { + "_searchInternal": Object { + "alternativeNames": Array [ + "0", + "0.js", + "0js", + ], + "expiresAt": Any, + "popularAlternativeNames": Array [], + }, + "bin": Object {}, + "changelogFilename": null, + "computedKeywords": Array [], + "computedMetadata": Object {}, + "created": NaN, + "dependencies": Object {}, + "dependents": 0, + "deprecated": false, + "deprecatedReason": null, + "description": null, + "devDependencies": Object {}, + "downloadsLast30Days": 0, + "downloadsRatio": 0, + "gitHead": null, + "githubRepo": null, + "homepage": null, + "humanDependents": "0", + "humanDownloadsLast30Days": "0", + "isDeprecated": false, + "isSecurityHeld": true, + "jsDelivrHits": 0, + "keywords": Array [], + "lastCrawl": Any, + "lastPublisher": null, + "license": null, + "modified": NaN, + "moduleTypes": Array [ + "unknown", + ], + "name": "0", + "objectID": "0", + "originalAuthor": Object { + "name": "npm", + }, + "owner": Object { + "avatar": "https://github.com/npm.png", + "link": "https://github.com/npm", + "name": "npm", + }, + "owners": Array [], + "popular": false, + "readme": "", + "repository": Object { + "branch": "master", + "head": undefined, + "host": "github.com", + "path": "", + "project": "security-holder", + "type": "git", + "url": "npm/security-holder", + "user": "npm", + }, + "rev": Any, + "styleTypes": Array [], + "tags": Object { + "latest": "1.2.3", + }, + "types": Object { + "ts": Object { + "possible": true, + }, + }, + "version": "0.0.0", + "versions": Object {}, +} +`; + +exports[`security held only log security held flag for the correct repo 1`] = ` +Object { + "_searchInternal": Object { + "alternativeNames": Array [ + "0", + "0.js", + "0js", + ], + "expiresAt": Any, + "popularAlternativeNames": Array [], + }, + "bin": Object {}, + "changelogFilename": null, + "computedKeywords": Array [], + "computedMetadata": Object {}, + "created": NaN, + "dependencies": Object {}, + "dependents": 0, + "deprecated": false, + "deprecatedReason": null, + "description": null, + "devDependencies": Object {}, + "downloadsLast30Days": 0, + "downloadsRatio": 0, + "gitHead": null, + "githubRepo": null, + "homepage": null, + "humanDependents": "0", + "humanDownloadsLast30Days": "0", + "isDeprecated": false, + "isSecurityHeld": false, + "jsDelivrHits": 0, + "keywords": Array [], + "lastCrawl": Any, + "lastPublisher": null, + "license": null, + "modified": NaN, + "moduleTypes": Array [ + "unknown", + ], + "name": "0", + "objectID": "0", + "originalAuthor": Object { + "name": "npm", + }, + "owner": Object { + "avatar": undefined, + "link": "https://gitlab.com/npm", + "name": "npm", + }, + "owners": Array [], + "popular": false, + "readme": "", + "repository": Object { + "branch": "master", + "head": undefined, + "host": "gitlab.com", + "path": "", + "project": "security-holder", + "type": "git", + "url": "gitlab:npm/security-holder", + "user": "npm", + }, + "rev": Any, + "styleTypes": Array [], + "tags": Object { + "latest": "1.2.3", + }, + "types": Object { + "ts": Object { + "possible": true, + }, + }, + "version": "0.0.0", + "versions": Object {}, +} +`; diff --git a/src/__tests__/formatPkg.test.ts b/src/__tests__/formatPkg.test.ts index 2545cec6a..b8e553a27 100644 --- a/src/__tests__/formatPkg.test.ts +++ b/src/__tests__/formatPkg.test.ts @@ -8,7 +8,7 @@ import { getVersions, getExportKeys, } from '../formatPkg'; -import type { GetPackage } from '../npm/types'; +import type { GetPackage, PackageRepo } from '../npm/types'; import preact from './preact-simplified.json'; import rawPackages from './rawPackages.json'; @@ -859,3 +859,57 @@ describe('deprecated', () => { }); }); }); + +describe('security held', () => { + it('log security held flag', () => { + const pkg: GetPackage = { + ...BASE, + 'dist-tags': { + latest: '1.2.3', + }, + versions: { + '1.2.3': { + ...BASE_VERSION, + }, + }, + repository: 'npm/security-holder' as unknown as PackageRepo, + author: { name: 'npm' }, + }; + const formatted = formatPkg(pkg); + + expect(formatted).toMatchSnapshot({ + rev: expect.any(String), + lastCrawl: expect.any(String), + isSecurityHeld: true, + _searchInternal: { + expiresAt: expect.any(Number), + }, + }); + }); + + it('only log security held flag for the correct repo', () => { + const pkg: GetPackage = { + ...BASE, + 'dist-tags': { + latest: '1.2.3', + }, + versions: { + '1.2.3': { + ...BASE_VERSION, + }, + }, + repository: 'gitlab:npm/security-holder' as unknown as PackageRepo, + author: { name: 'npm' }, + }; + const formatted = formatPkg(pkg); + + expect(formatted).toMatchSnapshot({ + rev: expect.any(String), + lastCrawl: expect.any(String), + isSecurityHeld: false, + _searchInternal: { + expiresAt: expect.any(Number), + }, + }); + }); +}); diff --git a/src/__tests__/saveDocs.test.ts b/src/__tests__/saveDocs.test.ts index fd9806d46..be76fffdb 100644 --- a/src/__tests__/saveDocs.test.ts +++ b/src/__tests__/saveDocs.test.ts @@ -7,6 +7,189 @@ import preact from './preact-simplified.json'; jest.setTimeout(15000); +const FINAL_BASE = { + _searchInternal: { + alternativeNames: ['preact', 'preact.js', 'preactjs'], + popularAlternativeNames: ['preact', 'preact.js', 'preactjs'], + downloadsMagnitude: 7, + expiresAt: '2021-08-10', + jsDelivrPopularity: 0, + }, + bin: {}, + changelogFilename: null, + computedKeywords: [], + computedMetadata: {}, + created: 1441939293521, + dependencies: {}, + dependents: 0, + deprecated: false, + deprecatedReason: null, + description: + 'Fast 3kb React alternative with the same modern API. Components & Virtual DOM.', + devDependencies: { + '@types/chai': '^4.1.7', + '@types/mocha': '^5.2.5', + '@types/node': '^9.6.40', + 'babel-cli': '^6.24.1', + 'babel-core': '^6.24.1', + 'babel-eslint': '^8.2.6', + 'babel-loader': '^7.0.0', + 'babel-plugin-transform-object-rest-spread': '^6.23.0', + 'babel-plugin-transform-react-jsx': '^6.24.1', + 'babel-preset-env': '^1.6.1', + bundlesize: '^0.17.0', + chai: '^4.2.0', + copyfiles: '^2.1.0', + 'core-js': '^2.6.0', + coveralls: '^3.0.0', + 'cross-env': '^5.1.4', + diff: '^3.0.0', + eslint: '^4.18.2', + 'eslint-plugin-react': '^7.11.1', + 'flow-bin': '^0.89.0', + 'gzip-size-cli': '^2.0.0', + 'istanbul-instrumenter-loader': '^3.0.0', + jscodeshift: '^0.5.0', + karma: '^3.1.3', + 'karma-babel-preprocessor': '^7.0.0', + 'karma-chai-sinon': '^0.1.5', + 'karma-chrome-launcher': '^2.2.0', + 'karma-coverage': '^1.1.2', + 'karma-mocha': '^1.3.0', + 'karma-mocha-reporter': '^2.2.5', + 'karma-sauce-launcher': '^1.2.0', + 'karma-sinon': '^1.0.5', + 'karma-source-map-support': '^1.3.0', + 'karma-sourcemap-loader': '^0.3.6', + 'karma-webpack': '^3.0.5', + mocha: '^5.0.4', + 'npm-run-all': '^4.1.5', + puppeteer: '^1.11.0', + rimraf: '^2.5.3', + rollup: '^0.57.1', + 'rollup-plugin-babel': '^3.0.2', + 'rollup-plugin-memory': '^3.0.0', + 'rollup-plugin-node-resolve': '^3.4.0', + sinon: '^4.4.2', + 'sinon-chai': '^3.3.0', + typescript: '^3.0.1', + 'uglify-js': '^2.7.5', + webpack: '^4.27.1', + }, + downloadsLast30Days: 2874638, + downloadsRatio: 0.0023, + gitHead: 'master', + githubRepo: { + head: 'master', + path: '', + project: 'preact', + user: 'developit', + }, + homepage: null, + humanDependents: '0', + humanDownloadsLast30Days: '2.9m', + isDeprecated: false, + jsDelivrHits: 0, + keywords: [ + 'preact', + 'react', + 'virtual dom', + 'vdom', + 'components', + 'virtual', + 'dom', + ], + lastCrawl: '2021-07-11T12:31:18.112Z', + lastPublisher: { + avatar: 'https://gravatar.com/avatar/ad82ff1463f3e3b7b4a44c5f499912ae', + email: 'npm.leah@hrmny.sh', + link: 'https://www.npmjs.com/~harmony', + name: 'harmony', + }, + license: 'MIT', + modified: 1564778088321, + moduleTypes: ['esm'], + name: 'preact', + objectID: 'preact', + originalAuthor: { + email: 'jason@developit.ca', + name: 'Jason Miller', + }, + owner: { + avatar: 'https://github.com/developit.png', + link: 'https://github.com/developit', + name: 'developit', + }, + owners: [ + { + avatar: 'https://gravatar.com/avatar/85ed8e6da2fbf39abeb4995189be324c', + email: 'jason@developit.ca', + link: 'https://www.npmjs.com/~developit', + name: 'developit', + }, + { + avatar: 'https://gravatar.com/avatar/52401c37bc5c4d54a051c619767fdbf8', + email: 'ulliftw@gmail.com', + link: 'https://www.npmjs.com/~harmony', + name: 'harmony', + }, + { + avatar: 'https://gravatar.com/avatar/308439e12701ef85245dc0632dd07c2a', + email: 'luke@lukeed.com', + link: 'https://www.npmjs.com/~lukeed', + name: 'lukeed', + }, + { + avatar: 'https://gravatar.com/avatar/4ed639a3ea6219b80b58e2e81ff9ba47', + email: 'marvin@marvinhagemeister.de', + link: 'https://www.npmjs.com/~marvinhagemeister', + name: 'marvinhagemeister', + }, + { + avatar: 'https://gravatar.com/avatar/83589d88ac76ddc2853562f9a817fe27', + email: 'prateek89born@gmail.com', + link: 'https://www.npmjs.com/~prateekbh', + name: 'prateekbh', + }, + { + avatar: 'https://gravatar.com/avatar/88747cce15801e9e96bcb76895fcd7f9', + email: 'hello@preactjs.com', + link: 'https://www.npmjs.com/~preactjs', + name: 'preactjs', + }, + { + avatar: 'https://gravatar.com/avatar/d279821c96bb49eeaef68b5456f42074', + email: 'allamsetty.anup@gmail.com', + link: 'https://www.npmjs.com/~reznord', + name: 'reznord', + }, + ], + popular: false, + readme: '', + repository: { + branch: 'master', + head: undefined, + host: 'github.com', + path: '', + project: 'preact', + type: 'git', + url: 'https://github.com/developit/preact', + user: 'developit', + }, + tags: { + latest: '8.5.0', + next: '10.0.0-rc.1', + }, + types: { + ts: 'included', + }, + version: '8.5.0', + versions: { + '10.0.0-rc.1': '2019-08-02T20:34:45.123Z', + '8.5.0': '2019-08-02T18:34:23.572Z', + }, +}; + describe('saveDoc', () => { it('should always produce the same records', async () => { const client = algoliasearch('e', ''); @@ -16,207 +199,157 @@ describe('saveDoc', () => { }); const final = { - _searchInternal: { - alternativeNames: ['preact', 'preact.js', 'preactjs'], - downloadsMagnitude: 7, - expiresAt: '2021-08-10', - jsDelivrPopularity: 0, - }, - bin: {}, - changelogFilename: null, - computedKeywords: [], - computedMetadata: {}, - created: 1441939293521, - dependencies: {}, - dependents: 0, - deprecated: false, - deprecatedReason: null, - description: - 'Fast 3kb React alternative with the same modern API. Components & Virtual DOM.', - devDependencies: { - '@types/chai': '^4.1.7', - '@types/mocha': '^5.2.5', - '@types/node': '^9.6.40', - 'babel-cli': '^6.24.1', - 'babel-core': '^6.24.1', - 'babel-eslint': '^8.2.6', - 'babel-loader': '^7.0.0', - 'babel-plugin-transform-object-rest-spread': '^6.23.0', - 'babel-plugin-transform-react-jsx': '^6.24.1', - 'babel-preset-env': '^1.6.1', - bundlesize: '^0.17.0', - chai: '^4.2.0', - copyfiles: '^2.1.0', - 'core-js': '^2.6.0', - coveralls: '^3.0.0', - 'cross-env': '^5.1.4', - diff: '^3.0.0', - eslint: '^4.18.2', - 'eslint-plugin-react': '^7.11.1', - 'flow-bin': '^0.89.0', - 'gzip-size-cli': '^2.0.0', - 'istanbul-instrumenter-loader': '^3.0.0', - jscodeshift: '^0.5.0', - karma: '^3.1.3', - 'karma-babel-preprocessor': '^7.0.0', - 'karma-chai-sinon': '^0.1.5', - 'karma-chrome-launcher': '^2.2.0', - 'karma-coverage': '^1.1.2', - 'karma-mocha': '^1.3.0', - 'karma-mocha-reporter': '^2.2.5', - 'karma-sauce-launcher': '^1.2.0', - 'karma-sinon': '^1.0.5', - 'karma-source-map-support': '^1.3.0', - 'karma-sourcemap-loader': '^0.3.6', - 'karma-webpack': '^3.0.5', - mocha: '^5.0.4', - 'npm-run-all': '^4.1.5', - puppeteer: '^1.11.0', - rimraf: '^2.5.3', - rollup: '^0.57.1', - 'rollup-plugin-babel': '^3.0.2', - 'rollup-plugin-memory': '^3.0.0', - 'rollup-plugin-node-resolve': '^3.4.0', - sinon: '^4.4.2', - 'sinon-chai': '^3.3.0', - typescript: '^3.0.1', - 'uglify-js': '^2.7.5', - webpack: '^4.27.1', - }, - downloadsLast30Days: 2874638, - downloadsRatio: 0.0023, - gitHead: 'master', - githubRepo: { - head: 'master', - path: '', - project: 'preact', - user: 'developit', - }, - homepage: null, - humanDependents: '0', - humanDownloadsLast30Days: '2.9m', - isDeprecated: false, - jsDelivrHits: 0, - keywords: [ - 'preact', - 'react', - 'virtual dom', - 'vdom', - 'components', - 'virtual', - 'dom', - ], - lastCrawl: '2021-07-11T12:31:18.112Z', - lastPublisher: { - avatar: 'https://gravatar.com/avatar/ad82ff1463f3e3b7b4a44c5f499912ae', - email: 'npm.leah@hrmny.sh', - link: 'https://www.npmjs.com/~harmony', - name: 'harmony', - }, - license: 'MIT', - modified: 1564778088321, - moduleTypes: ['esm'], - name: 'preact', - objectID: 'preact', - originalAuthor: { - email: 'jason@developit.ca', - name: 'Jason Miller', + ...FINAL_BASE, + }; + const clean = expect.objectContaining({ + ...final, + lastCrawl: expect.any(String), + downloadsLast30Days: expect.any(Number), + downloadsRatio: expect.any(Number), + humanDownloadsLast30Days: expect.any(String), + modified: expect.any(Number), + _searchInternal: expect.objectContaining({ + ...final._searchInternal, + downloadsMagnitude: expect.any(Number), + expiresAt: expect.any(Number), + }), + }); + + await saveDoc({ formatted: formatPkg(preact), index }); + + expect(index.saveObject).toHaveBeenCalledWith(clean); + }); + + it('should not add popular alternative names for non-popular packages', async () => { + const client = algoliasearch('e', ''); + const index = client.initIndex('a'); + jest.spyOn(index, 'saveObject').mockImplementationOnce(() => { + return true as any; + }); + + const final = { + ...FINAL_BASE, + name: 'reactjs', + objectID: 'reactjs', + tags: { + latest: '1.0.0', }, - owner: { - avatar: 'https://github.com/developit.png', - link: 'https://github.com/developit', - name: 'developit', + version: '1.0.0', + versions: { + '1.0.0': '2019-08-02T18:34:23.572Z', }, - owners: [ - { - avatar: - 'https://gravatar.com/avatar/85ed8e6da2fbf39abeb4995189be324c', - email: 'jason@developit.ca', - link: 'https://www.npmjs.com/~developit', - name: 'developit', - }, - { - avatar: - 'https://gravatar.com/avatar/52401c37bc5c4d54a051c619767fdbf8', - email: 'ulliftw@gmail.com', - link: 'https://www.npmjs.com/~harmony', - name: 'harmony', - }, - { - avatar: - 'https://gravatar.com/avatar/308439e12701ef85245dc0632dd07c2a', - email: 'luke@lukeed.com', - link: 'https://www.npmjs.com/~lukeed', - name: 'lukeed', - }, - { - avatar: - 'https://gravatar.com/avatar/4ed639a3ea6219b80b58e2e81ff9ba47', - email: 'marvin@marvinhagemeister.de', - link: 'https://www.npmjs.com/~marvinhagemeister', - name: 'marvinhagemeister', - }, - { - avatar: - 'https://gravatar.com/avatar/83589d88ac76ddc2853562f9a817fe27', - email: 'prateek89born@gmail.com', - link: 'https://www.npmjs.com/~prateekbh', - name: 'prateekbh', - }, - { - avatar: - 'https://gravatar.com/avatar/88747cce15801e9e96bcb76895fcd7f9', - email: 'hello@preactjs.com', - link: 'https://www.npmjs.com/~preactjs', - name: 'preactjs', + }; + const clean = expect.objectContaining({ + ...final, + lastCrawl: expect.any(String), + downloadsLast30Days: expect.any(Number), + downloadsRatio: expect.any(Number), + humanDownloadsLast30Days: expect.any(String), + modified: expect.any(Number), + _searchInternal: expect.objectContaining({ + popularAlternativeNames: [], + downloadsMagnitude: expect.any(Number), + expiresAt: expect.any(Number), + }), + }); + + await saveDoc({ + formatted: formatPkg({ + ...preact, + name: 'reactjs', + 'dist-tags': { latest: '1.0.0' }, + versions: { + '1.0.0': { + ...preact.versions['8.5.0'], + name: 'reactjs', + version: '1.0.0', + }, }, - { - avatar: - 'https://gravatar.com/avatar/d279821c96bb49eeaef68b5456f42074', - email: 'allamsetty.anup@gmail.com', - link: 'https://www.npmjs.com/~reznord', - name: 'reznord', + time: { + ...preact.time, + '1.0.0': '2019-08-02T18:34:23.572Z', }, - ], - popular: false, - readme: '', + }), + index, + }); + + expect(index.saveObject).toHaveBeenCalledWith(clean); + }); + + it('should skip getting extra data for security held packages', async () => { + const client = algoliasearch('e', ''); + const index = client.initIndex('a'); + jest.spyOn(index, 'saveObject').mockImplementationOnce(() => { + return true as any; + }); + + const final = { + ...FINAL_BASE, + name: 'trello-enterprises', + objectID: 'trello-enterprises', + tags: { + latest: '1000.1000.1000', + }, + version: '1000.1000.1000', + versions: { + '1000.1000.1000': '2019-08-02T18:34:23.572Z', + }, repository: { branch: 'master', head: undefined, host: 'github.com', path: '', - project: 'preact', + project: 'security-holder', type: 'git', - url: 'https://github.com/developit/preact', - user: 'developit', + url: 'https://github.com/npm/security-holder', + user: 'npm', }, - tags: { - latest: '8.5.0', - next: '10.0.0-rc.1', - }, - types: { - ts: 'included', - }, - version: '8.5.0', - versions: { - '10.0.0-rc.1': '2019-08-02T20:34:45.123Z', - '8.5.0': '2019-08-02T18:34:23.572Z', + githubRepo: { + head: 'master', + path: '', + project: 'security-holder', + user: 'npm', }, + downloadsLast30Days: 0, + humanDownloadsLast30Days: '0', + isSecurityHeld: true, }; const clean = expect.objectContaining({ ...final, + owner: expect.any(Object), + homepage: expect.any(String), lastCrawl: expect.any(String), - downloadsLast30Days: expect.any(Number), downloadsRatio: expect.any(Number), - humanDownloadsLast30Days: expect.any(String), modified: expect.any(Number), _searchInternal: expect.objectContaining({ - downloadsMagnitude: expect.any(Number), + popularAlternativeNames: [], expiresAt: expect.any(Number), }), }); - await saveDoc({ formatted: formatPkg(preact), index }); + await saveDoc({ + formatted: formatPkg({ + ...preact, + name: 'trello-enterprises', + 'dist-tags': { latest: '1000.1000.1000' }, + versions: { + '1000.1000.1000': { + ...preact.versions['8.5.0'], + name: 'trello-enterprises', + version: '1000.1000.1000', + }, + }, + time: { + ...preact.time, + '1000.1000.1000': '2019-08-02T18:34:23.572Z', + }, + repository: { + type: 'git', + url: 'https://github.com/npm/security-holder', + }, + }), + index, + }); expect(index.saveObject).toHaveBeenCalledWith(clean); }); diff --git a/src/config.ts b/src/config.ts index 42c3e71d2..35b7137d6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -18,6 +18,7 @@ const indexSettings: Settings = { '_searchInternal.expiresAt', 'deprecated', 'isDeprecated', + 'isSecurityHeld', 'types.ts', 'moduleTypes', 'styleTypes', @@ -38,6 +39,7 @@ const indexSettings: Settings = { 'words', 'proximity', 'attribute', + 'asc(isSecurityHeld)', 'asc(deprecated)', 'asc(isDeprecated)', 'asc(badPackage)', @@ -179,6 +181,8 @@ export const config = { retrySkipped: ms('1 minute'), retryBackoffPow: 3, refreshPeriod: ms('2 minutes'), + alternativeNamesNpmDownloadsThreshold: 5000, + alternativeNamesJsDelivrHitsThreshold: 10000, // Watch watchMaxPrefetch: 10, diff --git a/src/formatPkg.ts b/src/formatPkg.ts index 1e08022db..e42ceec9f 100644 --- a/src/formatPkg.ts +++ b/src/formatPkg.ts @@ -146,6 +146,10 @@ export function formatPkg(pkg: GetPackage): RawPkg | undefined { const tags = pkg['dist-tags']; const isDeprecated = cleaned.deprecated !== undefined && cleaned.deprecated !== false; + const isSecurityHeld = + repository?.host === 'github.com' && + repository?.user === 'npm' && + repository?.project === 'security-holder'; const rawPkg: RawPkg = { objectID: cleaned.name, @@ -154,6 +158,7 @@ export function formatPkg(pkg: GetPackage): RawPkg | undefined { downloadsLast30Days: 0, downloadsRatio: 0, humanDownloadsLast30Days: numeral(0).format('0.[0]a'), + jsDelivrHits: 0, popular: false, version, versions, @@ -170,6 +175,7 @@ export function formatPkg(pkg: GetPackage): RawPkg | undefined { deprecated: isDeprecated ? cleaned.deprecated! : false, isDeprecated, deprecatedReason: isDeprecated ? String(cleaned.deprecated) : null, + isSecurityHeld, homepage: getHomePage(cleaned), license, keywords, @@ -180,12 +186,16 @@ export function formatPkg(pkg: GetPackage): RawPkg | undefined { lastPublisher, owners: (cleaned.owners || []).map(formatUser), bin: cleaned.bin || {}, + humanDependents: '0', + dependents: 0, types, moduleTypes, styleTypes, + changelogFilename: null, lastCrawl: new Date().toISOString(), _searchInternal: { alternativeNames, + popularAlternativeNames: [], expiresAt: getExpiresAt(), }, }; diff --git a/src/npm/__tests__/index.test.ts b/src/npm/__tests__/index.test.ts index f280c3a90..c233a63f3 100644 --- a/src/npm/__tests__/index.test.ts +++ b/src/npm/__tests__/index.test.ts @@ -19,7 +19,7 @@ describe('findAll()', () => { expect.objectContaining({ id: '0', key: '0', - value: { rev: '9-2f99061d7a24f3ac9730d35566e66db9' }, + value: { rev: '10-b34ca87de9ad5692f1d0093f7752f331' }, }) ); }); diff --git a/src/saveDocs.ts b/src/saveDocs.ts index 9716a25b8..0afeafb7b 100644 --- a/src/saveDocs.ts +++ b/src/saveDocs.ts @@ -2,6 +2,7 @@ import type { SearchIndex } from 'algoliasearch'; import type { FinalPkg, RawPkg } from './@types/pkg'; import { getChangelog } from './changelog'; +import { config } from './config'; import * as jsDelivr from './jsDelivr'; import { getModuleTypes, getStyleTypes } from './jsDelivr/pkgTypes'; import * as npm from './npm'; @@ -27,6 +28,10 @@ export async function saveDoc({ } async function addMetaData(pkg: RawPkg): Promise { + if (pkg.isSecurityHeld) { + return pkg; + } + const [download, dependent, hit, filelist] = await Promise.all([ npm.getDownload(pkg), npm.getDependent(pkg), @@ -58,6 +63,19 @@ async function addMetaData(pkg: RawPkg): Promise { }, }; + const hasFewDownloads = + final.downloadsLast30Days <= config.alternativeNamesNpmDownloadsThreshold && + final.jsDelivrHits <= config.alternativeNamesJsDelivrHitsThreshold; + + const addPopularAlternativeNames = + final.popular || + (!final.isDeprecated && !final.isSecurityHeld && !hasFewDownloads); + + if (addPopularAlternativeNames) { + final._searchInternal.popularAlternativeNames = + final._searchInternal.alternativeNames; + } + datadog.timing('saveDocs.addMetaData.one', Date.now() - start); return final; }