refactor: use remote abbreviated version data (#1675)

avoid npm abbreviated version fields change closes #1667 and starting use the new domain: npmmirror.com
cnpm · Nov 7, 2021 · 39cf77a · 39cf77a
1 parent 37ba629
commit 39cf77a
Show file tree

Hide file tree

Showing 12 changed files with 111 additions and 46 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR ${CNPM_DIR}
 
 COPY package.json ${CNPM_DIR}
 
-RUN npm set registry https://registry.npm.taobao.org
+RUN npm set registry https://registry.npmmirror.com
 
 RUN npm install --production
 

diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ Our goal is to provide a low cost maintenance, easy to use, and easy to scale so
 ## What can you do with `cnpmjs.org`?
 
 * Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now)
-* Build a npm mirror. (we use it to build a mirror in China: [https://npm.taobao.org/](https://npm.taobao.org/))
+* Build a npm mirror. (we use it to build a mirror in China: [https://npmmirror.com/](https://npmmirror.com/))
 * Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)
 
 ## Features
@@ -66,7 +66,7 @@ you only need to change the registry in client config.
 
 * [node](http://nodejs.org) >= 8.0.0
 * Databases: only required one type
- * [sqlite3](https://npm.taobao.org/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
+ * [sqlite3](https://npmmirror.com/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
  * [MySQL](http://dev.mysql.com/downloads/) >= 5.6.16, include `mysqld` and `mysql cli`. I test on `[email protected]`.
  * MariaDB
  * PostgreSQL

diff --git a/config/index.js b/config/index.js
@@ -207,8 +207,8 @@ var config = {
  // sync source, upstream registry
  // If you want to directly sync from official npm's registry
  // please drop them an email first
- sourceNpmRegistry: 'https://registry.npm.taobao.org',
- sourceNpmWeb: 'https://npm.taobao.org',
+ sourceNpmRegistry: 'https://registry.npmmirror.com',
+ sourceNpmWeb: 'https://npmmirror.com',
 
  // upstream registry is base on cnpm/cnpmjs.org or not
  // if your upstream is official npm registry, please turn it off

diff --git a/controllers/registry/package/list.js b/controllers/registry/package/list.js
@@ -386,6 +386,13 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags
  continue;
  }
  // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
+ var hasInstallScript;
+ if (row.package.scripts) {
+ // https://www.npmjs.com/package/fix-has-install-script
+ if (row.package.scripts.install || row.package.scripts.preinstall || row.package.scripts.postinstall) {
+ hasInstallScript = true;
+ }
+ }
  var pkg = {
  name: row.package.name,
  version: row.package.version,
@@ -404,6 +411,7 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags
  engines: row.package.engines,
  workspaces: row.package.workspaces,
  _hasShrinkwrap: row.package._hasShrinkwrap,
+ hasInstallScript: hasInstallScript,
  publish_time: row.package.publish_time || row.publish_time,
  };
  common.setDownloadURL(pkg, ctx);

diff --git a/controllers/sync_module_worker.js b/controllers/sync_module_worker.js
@@ -841,6 +841,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
 
  // get package AbbreviatedMetadata
  var remoteAbbreviatedMetadatas = {};
+ // store remote abbreviated versions
+ var remoteAbbreviatedVersionsMap = {};
  if (config.enableAbbreviatedMetadata) {
  // use ?cache=0 tell registry dont use cache result
  var packageUrl = '/' + name.replace('/', '%2f') + '?cache=0&sync_timestamp=' + Date.now();
@@ -860,9 +862,9 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  name, err, result.headers, result.data);
  }
  if (data) {
- var versions = data && data.versions || {};
- for (var version in versions) {
- const item = versions[version];
+ remoteAbbreviatedVersionsMap = data && data.versions || {};
+ for (var version in remoteAbbreviatedVersionsMap) {
+ const item = remoteAbbreviatedVersionsMap[version];
  if (!item) {
  continue;
  }
@@ -874,7 +876,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  metaData._hasShrinkwrap = item._hasShrinkwrap;
  }
 
- const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'workspaces' ];
+ // https://github.com/cnpm/cnpmjs.org/issues/1667
+ const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'workspaces', 'hasInstallScript' ];
  for (const key of metaDataKeys) {
  if (key in item) {
  hasMetaData = true;
@@ -1079,8 +1082,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  if (abbreviatedMetadata) {
  for (var key in abbreviatedMetadata) {
  const value = abbreviatedMetadata[key];
- // boolean: _hasShrinkwrap
- if (key === '_hasShrinkwrap' && typeof value === 'boolean') {
+ // boolean: _hasShrinkwrap, hasInstallScript
+ if ((key === '_hasShrinkwrap' || key === 'hasInstallScript') && typeof value === 'boolean') {
  if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) {
  missingAbbreviatedMetadatas.push(Object.assign({
  id: exists.id,
@@ -1171,7 +1174,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  var tries = 3;
  while (true) {
  try {
- yield that._syncOneVersion(index, syncModule);
+ yield that._syncOneVersion(index, syncModule, remoteAbbreviatedVersionsMap[syncModule.version]);
  syncedVersionNames.push(syncModule.version);
  break;
  } catch (err) {
@@ -1322,7 +1325,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  that.log(' [%s] saving %d missing moduleAbbreviateds', name, missingModuleAbbreviateds.length);
 
  var res = yield gather(missingModuleAbbreviateds.map(function (item) {
- return packageService.saveModuleAbbreviated(item);
+ return packageService.saveModuleAbbreviated(item, remoteAbbreviatedVersionsMap[item.version]);
  }));
 
  for (var i = 0; i < res.length; i++) {
@@ -1541,7 +1544,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  return syncedVersionNames;
 };
 
-SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
+SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage, remoteAbbreviatedVersion) {
  var delay = Date.now() - sourcePackage.publish_time;
  logger.syncInfo('[sync_module_worker] delay: %s ms, publish_time: %s, start sync %s@%s',
  delay, utility.logDate(new Date(sourcePackage.publish_time)),
@@ -1678,7 +1681,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
  throw err;
  }
  logger.syncInfo('[sync_module_worker] uploaded, saving %j to database', result);
- var r = yield afterUpload(result);
+ var r = yield afterUpload(result, remoteAbbreviatedVersion);
  logger.syncInfo('[sync_module_worker] sync %s@%s done!',
  sourcePackage.name, sourcePackage.version);
  return r;
@@ -1687,7 +1690,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
  fs.unlink(filepath, utility.noop);
  }
 
- function *afterUpload(result) {
+ function *afterUpload(result, remoteAbbreviatedVersion) {
  //make sure sync module have the correct author info
  //only if can not get maintainers, use the username
  var author = username;
@@ -1731,7 +1734,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
  var r = yield packageService.saveModule(mod);
  var moduleAbbreviatedId = null;
  if (config.enableAbbreviatedMetadata) {
- var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod);
+ var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod, remoteAbbreviatedVersion);
  moduleAbbreviatedId = moduleAbbreviatedResult.id;
  }
 

diff --git a/docs/dockerize/config.js b/docs/dockerize/config.js
@@ -168,7 +168,7 @@ var config = {
  // sync source, upstream registry
  // If you want to directly sync from official npm's registry
  // please drop them an email first
- sourceNpmRegistry: 'https://registry.npm.taobao.org',
+ sourceNpmRegistry: 'https://registry.npmmirror.com',
 
  // upstream registry is base on cnpm/cnpmjs.org or not
  // if your upstream is official npm registry, please turn it off

diff --git a/docs/network.puml b/docs/network.puml
@@ -1,6 +1,6 @@
 @startuml
 
-title cnpmjs.org, npm.taobao.org Network
+title cnpmjs.org, npmmirror.com Network
 
 node "China User" {
  [cnpm cli]
@@ -11,8 +11,8 @@ node "OSS: aliyuncs.com" {
 }
 
 node "SLB: 114.55.80.225 Hangzhou" {
- [npm.taobao.org]
- [registry.npm.taobao.org]
+ [npmmirror.com]
+ [registry.npmmirror.com]
 }
 
 node "npmtaobao3: 121.41.*" {
@@ -38,7 +38,7 @@ node "cnpm6: 47.88.189.193 Singapore" {
 }
 
 node "Aliyun CDN" {
- [cdn.npm.taobao.org]
+ [oss.npmmirror.com]
 }
 
 node "qiniu CDN" {
@@ -59,17 +59,17 @@ node "npmjs.com" {
  [www.npmjs.com]
 }
 
-[cnpm cli] -down-> [registry.npm.taobao.org]: Read, Write
-[cnpm cli] -down-> [cdn.npm.taobao.org]: Read tgz
-[cnpm cli] -down-> [npm.taobao.org]: "Read /mirrors"
+[cnpm cli] -down-> [registry.npmmirror.com]: Read, Write
+[cnpm cli] -down-> [oss.npmmirror.com]: Read tgz
+[cnpm cli] -down-> [npmmirror.com]: "Read /mirrors"
 
-[registry.npm.taobao.org] -down-> [nginx 5]
+[registry.npmmirror.com] -down-> [nginx 5]
 [nginx 5] -down-> [registry 5]
-[npm.taobao.org] -down-> [nginx 5]
+[npmmirror.com] -down-> [nginx 5]
 [nginx 5] -down-> [web 5]
-[registry.npm.taobao.org] -down-> [nginx 6]
+[registry.npmmirror.com] -down-> [nginx 6]
 [nginx 6] -down-> [registry 6]
-[npm.taobao.org] -down-> [nginx 6]
+[npmmirror.com] -down-> [nginx 6]
 [nginx 6] -down-> [web 6]
 
 [registry 5] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read, Write
@@ -92,7 +92,7 @@ node "npmjs.com" {
 [syncer 7] -down-> [cnpmjs.up.qiniu.com]: Write
 [syncer 7] -down-> [registry.npmjs.com]: Read
 
-[cdn.npm.taobao.org] -down-> [tnpm-hz.oss-cn-hangzhou]: Read
+[oss.npmmirror.com] -down-> [tnpm-hz.oss-cn-hangzhou]: Read
 [dn-cnpm.qbox.me] -down-> [cnpmjs.up.qiniu.com]: Read
 
 @enduml
diff --git a/docs/web/readme.md b/docs/web/readme.md
@@ -7,7 +7,7 @@ So `cnpm` is meaning: **Company npm**.
 - Our public registry: [r.cnpmjs.org](https://r.cnpmjs.org), syncing from [registry.npmjs.com](https://registry.npmjs.com)
 - [cnpmjs.org](/) version: <span id="app-version"></span>
 - [Node.js](https://nodejs.org) version: <span id="node-version"></span>
-- For developers in China, please visit [the China mirror](https://npm.taobao.org). 中国用户请访问[国内镜像站点](https://npm.taobao.org)。
+- For developers in China, please visit [the China mirror](https://npmmirror.com). 中国用户请访问[国内镜像站点](https://npmmirror.com)。
 - Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)
 
 <div class="ant-table">
@@ -81,21 +81,21 @@ Badge URL: `https://cnpmjs.org/badge/d/cnpmjs.org.svg` ![cnpmjs.org-download-bad
 use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
 
 ```bash
-$ npm install -g cnpm --registry=https://registry.npm.taobao.org
+$ npm install -g cnpm --registry=https://registry.npmmirror.com
 ```
 
 Or you can alias NPM to use it:
 
 ```bash
-alias cnpm="npm --registry=https://registry.npm.taobao.org \
+alias cnpm="npm --registry=https://registry.npmmirror.com \
 --cache=$HOME/.npm/.cache/cnpm \
---disturl=https://npm.taobao.org/mirrors/node \
+--disturl=https://npmmirror.com/mirrors/node \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npm.taobao.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npmmirror.com \
  --cache=$HOME/.npm/.cache/cnpm \
- --disturl=https://npm.taobao.org/mirrors/node \
+ --disturl=https://npmmirror.com/mirrors/node \
  --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
@@ -118,7 +118,7 @@ $ cnpm sync connect
 sync package on web: [sync/connect](/sync/connect)
 
 ```bash
-$ open http://registry.npm.taobao.org/sync/connect
+$ open http://registry.npmmirror.com/sync/connect
 ```
 
 ### publish / unpublish
@@ -146,7 +146,7 @@ $ cnpm info cnpm
 
 Release [History](/history).
 
-## npmjs.org, cnpmjs.org and npm.taobao.org relation
+## npmjs.org, cnpmjs.org and npmmirror.com relation
 
 ![npm&cnpm](https://cloud.githubusercontent.com/assets/543405/21505401/fd0b6220-cca1-11e6-86ed-599cc81bb03b.png)
 

diff --git a/services/package.js b/services/package.js
@@ -355,11 +355,22 @@ exports.findAllModuleAbbreviateds = function* (where, order, limit, offset) {
 };
 
 // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
-exports.saveModuleAbbreviated = function* (mod) {
- var pkg = JSON.stringify(Object.assign({}, mod.package, {
- // ignore readme force
- readme: undefined,
- }));
+exports.saveModuleAbbreviated = function* (mod, remoteAbbreviatedVersion) {
+ // try to use remoteAbbreviatedVersion first
+ var pkg;
+ if (remoteAbbreviatedVersion) {
+ pkg = Object.assign({}, remoteAbbreviatedVersion, {
+ // override remote tarball
+ dist: Object.assign({}, remoteAbbreviatedVersion.dist, mod.package.dist, {
+ noattachment: undefined,
+ }),
+ });
+ } else {
+ pkg = Object.assign({}, mod.package, {
+ // ignore readme force
+ readme: undefined,
+ });
+ }
  var publish_time = mod.publish_time || Date.now();
  var item = yield models.ModuleAbbreviated.findByNameAndVersion(mod.name, mod.version);
  if (!item) {
@@ -369,7 +380,7 @@ exports.saveModuleAbbreviated = function* (mod) {
  });
  }
  item.publish_time = publish_time;
- item.package = pkg;
+ item.package = JSON.stringify(pkg);
 
  if (item.changed()) {
  item = yield item.save();

diff --git a/test/controllers/sync_module_worker.test.js b/test/controllers/sync_module_worker.test.js
@@ -257,6 +257,49 @@ describe('test/controllers/sync_module_worker.test.js', () => {
  assert(newEtag == lastResHeaders.etag);
  });
 
+ it('should sync mk2test-module-cnpmsync-issue-1667 with remoteAbbreviatedVersion success', function* () {
+ mm(config, 'enableAbbreviatedMetadata', true);
+ mm(config, 'sourceNpmRegistry', 'https://registry.npmjs.com');
+ var log = yield logService.create({
+ name: 'mk2test-module-cnpmsync-issue-1667',
+ username: 'fengmk2',
+ });
+ log.id.should.above(0);
+ var worker = new SyncModuleWorker({
+ logId: log.id,
+ name: 'mk2test-module-cnpmsync-issue-1667',
+ username: 'fengmk2',
+ noDep: true,
+ });
+ worker.start();
+ var end = thunkify.event(worker, 'end');
+ yield end();
+
+ let pkg;
+ let pkgV2;
+ let pkgV3;
+ let lastResHeaders;
+ function checkResult() {
+ return function (done) {
+ request(app)
+ .get('/mk2test-module-cnpmsync-issue-1667')
+ .set('accept', 'application/vnd.npm.install-v1+json')
+ .expect(function (res) {
+ lastResHeaders = res.headers;
+ console.log('%j', res.body);
+ pkg = res.body.versions['3.0.0'];
+ assert(pkg.hasInstallScript === true);
+ // no scripts
+ assert(!pkg.scripts);
+ assert(pkg.dist.key === '/mk2test-module-cnpmsync-issue-1667/-/mk2test-module-cnpmsync-issue-1667-3.0.0.tgz');
+ assert(!('noattachment' in pkg.dist));
+ })
+ .expect(200, done);
+ };
+ }
+ yield checkResult();
+ });
+
  it('should sync upstream first', function* () {
  mm(config, 'sourceNpmRegistryIsCNpm', true);
  var log = yield logService.create({

diff --git a/test/fixtures/scope-package/Makefile b/test/fixtures/scope-package/Makefile
@@ -4,7 +4,7 @@ TIMEOUT = 1000
 MOCHA_OPTS =
 
 install:
- @npm install --registry=https://registry.npm.taobao.org --disturl=https://npm.taobao.org/dist
+ @npm install --registry=https://registry.npmmirror.com --disturl=https://npmmirror.com/dist
 
 jshint: install
  @./node_modules/.bin/jshint .

diff --git a/test/mocks/package/koa/default.js b/test/mocks/package/koa/default.js
@@ -5,7 +5,7 @@ module.exports = {
  name: 'koa',
  version: '0.13.0',
  description: 'description: Koa web app framework',
- registryUrl: 'https://registry.npm.taobao.org/koa',
+ registryUrl: 'https://registry.npmmirror.com/koa',
  engines: {
  node: {
  version: '>= 0.11.13',