Skip to content

Commit

Permalink
feat: add SSR for robots
Browse files Browse the repository at this point in the history
  • Loading branch information
@zensh
zensh committed Oct 1, 2023
1 parent dd85afd commit 896c612
Show file tree
Hide file tree
Showing 13 changed files with 690 additions and 118 deletions.
52 changes: 26 additions & 26 deletions .eslintrc.cjs
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,29 @@
/* eslint-env node */
module.exports = {
env: {
browser: true,
es2021: true
},
extends: [
"plugin:@typescript-eslint/recommended-type-checked",
"plugin:@typescript-eslint/stylistic-type-checked"
],
plugins: ["@typescript-eslint"],
parser: "@typescript-eslint/parser",
parserOptions: {
project: ["./tsconfig.eslint.json"],
ecmaVersion: "latest",
sourceType: "module"
},
rules: {
// Note: you must disable the base rule as it can report incorrect errors
"space-before-function-paren": "off",
"@typescript-eslint/space-before-function-paren": "off",
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-unsafe-assignment": "off",
"@typescript-eslint/no-unsafe-return": "off",
"@typescript-eslint/no-unsafe-member-access": "off",
"@typescript-eslint/no-unsafe-argument": "off",
},
root: true,
env: {
browser: true,
es2022: true,
},
extends: [
'plugin:@typescript-eslint/recommended-type-checked',
'plugin:@typescript-eslint/stylistic-type-checked',
],
plugins: ['@typescript-eslint'],
parser: '@typescript-eslint/parser',
parserOptions: {
project: ['./tsconfig.eslint.json'],
ecmaVersion: 'latest',
sourceType: 'module',
},
rules: {
// Note: you must disable the base rule as it can report incorrect errors
'space-before-function-paren': 'off',
'@typescript-eslint/space-before-function-paren': 'off',
'@typescript-eslint/no-explicit-any': 'off',
'@typescript-eslint/no-unsafe-assignment': 'off',
'@typescript-eslint/no-unsafe-return': 'off',
'@typescript-eslint/no-unsafe-member-access': 'off',
'@typescript-eslint/no-unsafe-argument': 'off',
},
root: true,
}
1 change: 1 addition & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ RUN pnpm install --prod \
# for most source file changes.
COPY --chown=myuser dist ./dist/
COPY --chown=myuser config ./config/
COPY --chown=myuser html ./html/

ENV NODE_ENV=production
# Run the image.
Expand Down
5 changes: 4 additions & 1 deletion config/default.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,8 @@
"contactPoints": [
"127.0.0.1:9042"
]
}
},
"siteBase": "http://127.0.0.1:8080",
"writingBase": "http://127.0.0.1:8080",
"userBase": "http://127.0.0.1:8080"
}
16 changes: 14 additions & 2 deletions dist/app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,23 @@ import Router from '@koa/router';
import { encode } from 'cborg';
import { LogLevel, createLog, writeLog } from './log.js';
import { connect } from './db/scylladb.js';
import { versionAPI, healthzAPI, scrapingAPI, searchAPI, documentAPI, convertingAPI, } from './api.js';
import { healthzAPI, scrapingAPI, searchAPI, documentAPI, convertingAPI, } from './api.js';
import { renderIndex, renderPublication, renderGroup } from './ssr.js';
const GZIP_MIN_LENGTH = 128;
export async function initApp(app) {
// attach stateful components to the application context
app.context.db = await connect('ywws');
// create routes
const router = new Router();
router.use(initContext);
router.get('/', versionAPI);
router.get('/', renderIndex);
router.get('/healthz', healthzAPI);
router.get('/v1/scraping', scrapingAPI);
router.get('/v1/search', searchAPI);
router.get('/v1/document', documentAPI);
router.post('/v1/converting', convertingAPI);
router.get('/pub/:id', renderPublication);
router.get('/group/:id', renderGroup);
app.use(router.routes());
app.use(router.allowedMethods());
}
Expand Down Expand Up @@ -99,4 +102,13 @@ async function initContext(ctx, next) {
}
ctx.body = body;
}
else if (typeof body === 'string') {
if (body.length > GZIP_MIN_LENGTH &&
ctx.acceptsEncodings('gzip') === 'gzip') {
log.beforeGzip = body.length;
ctx.body = gzipSync(Buffer.from(body, 'utf-8'));
ctx.remove('Content-Length');
ctx.set('content-encoding', 'gzip');
}
}
}
207 changes: 207 additions & 0 deletions dist/ssr.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
import { readFileSync } from 'node:fs';
import { URL } from 'node:url';
import config from 'config';
import { decode, encode } from 'cborg';
import { Xid } from 'xid-ts';
import * as cheerio from 'cheerio';
import createError from 'http-errors';
import { toHTML } from './tiptap.js';
const indexTpl = readFileSync('./html/index.html', 'utf-8');
const publicationTpl = readFileSync('./html/publication.html', 'utf-8');
const groupTpl = readFileSync('./html/group.html', 'utf-8');
const siteBase = config.get('siteBase');
const writingBase = config.get('writingBase');
const userBase = config.get('userBase');
export async function renderIndex(ctx) {
const ctxheaders = {
'x-request-id': ctx.get('x-request-id'),
'x-auth-user': '000000000000000anon0',
'x-auth-user-rating': ctx.get('x-auth-user-rating'),
'x-auth-app': ctx.get('x-auth-app'),
'x-language': ctx.get('x-language'),
};
const $ = cheerio.load(indexTpl);
try {
const docs = await listIndex(ctxheaders);
for (const doc of docs) {
const cid = Xid.fromValue(doc.cid).toString();
const docUrl = `${siteBase}/pub/${cid}?gid=${Xid.fromValue(doc.gid).toString()}`;
$('ul').append(`<li><a id="${cid}" href="${docUrl}" target="_blank"></a></li>`);
$(`#${cid}`).text(doc.title);
}
}
catch (err) {
ctx.status = 404;
const url = ctx.get('x-request-url');
if (url !== '') {
$('#content').text(url + ' not found');
}
}
ctx.vary('Accept-Language');
ctx.type = 'text/html';
ctx.body = $.html();
}
export async function renderPublication(ctx) {
const ctxheaders = {
'x-request-id': ctx.get('x-request-id'),
'x-auth-user': '000000000000000anon0',
'x-auth-user-rating': ctx.get('x-auth-user-rating'),
'x-auth-app': ctx.get('x-auth-app'),
'x-language': ctx.get('x-language'),
};
const cid = ctx.params.id;
const { gid, language } = ctx.query;
const $ = cheerio.load(publicationTpl);
try {
const doc = await getPublication(ctxheaders, cid, (gid ?? ''), (language ?? ''));
const docUrl = `${siteBase}/pub/${Xid.fromValue(doc.cid).toString()}`;
const groupUrl = `${siteBase}/group/${Xid.fromValue(doc.gid).toString()}`;
$('html').prop('lang', doc.language);
$('meta[property="og:title"]').prop('content', doc.title);
$('meta[property="og:url"]').prop('content', docUrl);
$('#title').text(doc.title);
const authors = $('#authors');
authors.prop('href', groupUrl);
authors.text(groupUrl);
if (doc.authors != null && doc.authors.length > 0) {
authors.text(doc.authors.join(', '));
}
const updated_at = new Date(doc.updated_at).toUTCString();
$('#updated_time').text(updated_at);
$('#version').text(doc.version.toString());
const content = decode(doc.content);
$('#content').html(toHTML(content) +
`\n<p><a href="${docUrl}" target="_blank">${docUrl}</a></p>`);
ctx.set('last-modified', updated_at);
}
catch (err) {
ctx.status = 404;
const url = ctx.get('x-request-url');
if (url !== '') {
$('#content').text(url + ' not found');
}
}
ctx.vary('Accept-Language');
ctx.type = 'text/html';
ctx.body = $.html();
}
export async function renderGroup(ctx) {
const ctxheaders = {
'x-request-id': ctx.get('x-request-id'),
'x-auth-user': '000000000000000anon0',
'x-auth-user-rating': ctx.get('x-auth-user-rating'),
'x-auth-app': ctx.get('x-auth-app'),
'x-language': ctx.get('x-language'),
};
const gid = ctx.params.id;
const $ = cheerio.load(groupTpl);
try {
const group = await getGroup(ctxheaders, gid);
const groupUrl = `${siteBase}/group/${Xid.fromValue(group.id).toString()}`;
$('meta[property="og:title"]').prop('content', group.name);
$('meta[property="og:description"]').prop('content', group.slogan);
$('meta[property="og:url"]').prop('content', groupUrl);
$('#group_name').text(group.name);
$('#group_slogan').text(group.slogan);
const docs = await listPublications(ctxheaders, Xid.fromValue(group.id));
for (const doc of docs) {
const cid = Xid.fromValue(doc.cid).toString();
const docUrl = `${siteBase}/pub/${cid}?gid=${Xid.fromValue(doc.gid).toString()}`;
$('ul').append(`<li><a id="${cid}" href="${docUrl}" target="_blank"></a></li>`);
$(`#${cid}`).text(doc.title);
}
}
catch (err) {
ctx.status = 404;
const url = ctx.get('x-request-url');
if (url !== '') {
$('#content').text(url + ' not found');
}
}
ctx.vary('Accept-Language');
ctx.type = 'text/html';
ctx.body = $.html();
}
async function getGroup(headers, gid) {
const api = new URL('/v1/group', userBase);
if (isXid(gid)) {
api.searchParams.append('id', gid);
}
else {
api.searchParams.append('cn', gid);
}
api.searchParams.append('fields', 'cn,name,status,slogan');
headers.accept = 'application/cbor';
const res = await fetch(api, {
headers,
});
if (res.status !== 200) {
throw createError(res.status, await res.text());
}
const data = await res.arrayBuffer();
const obj = decode(Buffer.from(data));
return obj.result;
}
async function getPublication(headers, cid, gid, language) {
const api = new URL('/v1/publication/implicit_get', writingBase);
api.searchParams.append('cid', cid);
if (gid !== '') {
api.searchParams.append('gid', gid);
}
if (language !== '') {
api.searchParams.append('language', language);
}
api.searchParams.append('fields', 'title,updated_at,authors,content');
headers.accept = 'application/cbor';
const res = await fetch(api, {
headers,
});
if (res.status !== 200) {
throw createError(res.status, await res.text());
}
const data = await res.arrayBuffer();
const obj = decode(Buffer.from(data));
return obj.result;
}
async function listPublications(headers, gid) {
const api = new URL('/v1/publication/list', writingBase);
headers.accept = 'application/cbor';
headers['content-type'] = 'application/cbor';
const res = await fetch(api, {
method: 'POST',
headers,
body: Buffer.from(encode({
gid: gid.toBytes(),
status: 2,
fields: ['title', 'updated_at'],
})),
});
if (res.status !== 200) {
throw createError(res.status, await res.text());
}
const data = await res.arrayBuffer();
const obj = decode(Buffer.from(data));
return obj.result;
}
async function listIndex(headers) {
const api = new URL('/v1/search?q=', writingBase);
headers.accept = 'application/cbor';
headers['content-type'] = 'application/cbor';
const res = await fetch(api, {
headers,
});
if (res.status !== 200) {
throw createError(res.status, await res.text());
}
const data = await res.arrayBuffer();
const obj = decode(Buffer.from(data));
return obj.result.hits;
}
function isXid(id) {
try {
Xid.parse(id);
return true;
}
catch (e) { }
return false;
}
Loading

0 comments on commit 896c612

Please sign in to comment.