Skip to content

Commit

Permalink
fix: fix collection SSR
Browse files Browse the repository at this point in the history
  • Loading branch information
@zensh
zensh committed Dec 2, 2023
1 parent b9f0e63 commit af5379c
Show file tree
Hide file tree
Showing 8 changed files with 453 additions and 71 deletions.
11 changes: 8 additions & 3 deletions dist/app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { encode } from 'cborg';
import { LogLevel, createLog, writeLog } from './log.js';
import { connect } from './db/scylladb.js';
import { healthzAPI, scrapingAPI, searchAPI, documentAPI, convertingAPI, } from './api.js';
import { renderIndex, renderPublication, renderGroup } from './ssr.js';
import { renderIndex, renderPublication, renderGroup, renderCollection, } from './ssr.js';
const GZIP_MIN_LENGTH = 128;
export async function initApp(app) {
// attach stateful components to the application context
Expand All @@ -18,8 +18,13 @@ export async function initApp(app) {
router.get('/v1/search', searchAPI);
router.get('/v1/document', documentAPI);
router.post('/v1/converting', convertingAPI);
router.get('/pub/:id', renderPublication);
router.get('/group/:id', renderGroup);
router.get('/pub/:cid', renderPublication);
router.get('/group/:gid', renderGroup);
router.get('/group/:gid/collection', renderCollection);
router.all('/:other+', (ctx) => {
ctx.redirect('/');
ctx.status = 307;
});
app.use(router.routes());
app.use(router.allowedMethods());
}
Expand Down
190 changes: 160 additions & 30 deletions dist/ssr.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ import { toHTML } from './tiptap.js';
import { lang639_3, isRTL } from './lang.js';
const ZeroID = Xid.default().toString();
const indexTpl = readFileSync('./html/index.html', 'utf-8');
const publicationTpl = readFileSync('./html/publication.html', 'utf-8');
const groupTpl = readFileSync('./html/group.html', 'utf-8');
const publicationTpl = readFileSync('./html/publication.html', 'utf-8');
const collectionTpl = readFileSync('./html/collection.html', 'utf-8');
const siteBase = config.get('siteBase');
const writingBase = config.get('writingBase');
const userBase = config.get('userBase');
Expand Down Expand Up @@ -44,9 +45,9 @@ export async function renderIndex(ctx) {
const headers = ctxHeaders(ctx);
const $ = cheerio.load(indexTpl);
const lang = headers['x-language'] ?? 'eng';
const info = metaInfos[lang] || metaInfos.eng;
$('title').text(info.title);
$('meta[name="description"]').prop('content', info.desc);
const siteInfo = metaInfos[lang] || metaInfos.eng;
$('title').text(siteInfo.title);
$('meta[name="description"]').prop('content', siteInfo.desc);
try {
await Promise.all([
(async () => {
Expand All @@ -72,40 +73,49 @@ export async function renderIndex(ctx) {
}
export async function renderPublication(ctx) {
const headers = ctxHeaders(ctx);
const cid = ctx.params.id;
const cid = ctx.params.cid;
const { gid, language } = ctx.query;
const $ = cheerio.load(publicationTpl);
const lang = headers['x-language'] ?? 'eng';
const info = metaInfos[lang];
if (info) {
$('title').text(info.title);
$('meta[name="description"]').prop('content', info.desc);
const siteInfo = metaInfos[lang];
if (siteInfo) {
$('title').text(siteInfo.title);
$('meta[name="description"]').prop('content', siteInfo.desc);
}
try {
const docs = await listPublished(headers, Xid.fromValue(cid));
renderPublicationItems($, docs);
const doc = await getPublication(headers, cid, (gid ?? ''), (language ?? lang));
const docs = await listPublished(headers, Xid.fromValue(cid));
renderPublicationItems($, docs.filter((item) => item.language !== doc.language));
const docUrl = `${siteBase}/pub/${Xid.fromValue(doc.cid).toString()}`;
const groupUrl = `${siteBase}/group/${Xid.fromValue(doc.gid).toString()}`;
$('html').prop('lang', doc.language);
if (isRTL(doc.language)) {
$('html').prop('dir', 'rtl');
}
$('meta[property="og:title"]').prop('content', doc.title);
$('meta[property="og:url"]').prop('content', docUrl);
$('meta[property="og:title"]').prop('content', doc.title);
if (doc.summary) {
$('meta[property="og:description"]').prop('content', doc.summary);
}
$('#title').text(doc.title);
const authors = $('#authors');
authors.prop('href', groupUrl);
authors.text(groupUrl);
if (doc.authors != null && doc.authors.length > 0) {
authors.text(doc.authors.join(', '));
if (doc.summary) {
$('#summary').text(doc.summary);
}
if (doc.authors) {
doc.authors.forEach((author) => $(`<span>${author}</span>`).appendTo(`#authors`));
}
if (doc.keywords) {
doc.keywords.forEach((keyword) => $(`<span>${keyword}</span>`).appendTo(`#keywords`));
}
const groupInfo = $('#group');
groupInfo.prop('href', groupUrl);
groupInfo.text(`Group: ${groupUrl}`);
const updated_at = new Date(doc.updated_at).toUTCString();
$('#updated_time').text(updated_at);
$('#version').text(doc.version.toString());
$('#updated_time').text(`Updated: ${updated_at}`);
$('#version').text(`Version: ${doc.version}`);
const content = decode(doc.content);
let contentHtml = toHTML(content) +
`\n<p><a title="Permalink" href="${docUrl}" target="_blank">${docUrl}</a></p>`;
`\n<p><a title="Permalink" href="${docUrl}" target="_blank">Permalink: ${docUrl}</a></p>`;
if (doc.rfp?.creation) {
contentHtml += `\n<p>Request For Payment, Price: ${doc.rfp.creation.price} WEN</p>`;
}
Expand All @@ -123,32 +133,97 @@ export async function renderPublication(ctx) {
ctx.type = 'text/html';
ctx.body = $.html();
}
export async function renderCollection(ctx) {
const headers = ctxHeaders(ctx);
// const gid = ctx.params.gid as string
const { cid: _cid } = ctx.query;
const $ = cheerio.load(collectionTpl);
const lang = headers['x-language'] ?? 'eng';
const siteInfo = metaInfos[lang];
if (siteInfo) {
$('title').text(siteInfo.title);
$('meta[name="description"]').prop('content', siteInfo.desc);
}
try {
const doc = await getCollection(headers, _cid);
const [language, info] = getCollectionInfo(doc, lang) ?? [];
if (!info || !language) {
throw createError(404, 'collection not found');
}
const gid = Xid.fromValue(doc.gid);
const cid = Xid.fromValue(doc.id);
const groupUrl = `${siteBase}/group/${gid.toString()}`;
const docUrl = `${groupUrl}?cid=${cid.toString()}`;
$('html').prop('lang', language);
if (isRTL(language)) {
$('html').prop('dir', 'rtl');
}
$('meta[property="og:url"]').prop('content', docUrl);
$('meta[property="og:title"]').prop('content', info.title);
if (info.summary) {
$('meta[property="og:description"]').prop('content', info.summary);
}
$('#title').text(info.title);
if (info.summary) {
$('#summary').text(info.summary);
}
if (info.authors) {
info.authors.forEach((author) => $(`<span>${author}</span>`).appendTo(`#authors`));
}
if (info.keywords) {
info.keywords.forEach((keyword) => $(`<span>${keyword}</span>`).appendTo(`#keywords`));
}
const groupInfo = $('#group');
groupInfo.prop('href', groupUrl);
groupInfo.text(`Group: ${groupUrl}`);
const updated_at = new Date(doc.updated_at).toUTCString();
$('#updated_time').text(`Updated: ${updated_at}`);
ctx.set('last-modified', updated_at);
try {
const docs = await listCollectionChildren(headers, cid);
renderCollectionChildrenItems($, docs);
}
catch (err) {
ignoreError();
}
}
catch (err) {
ctx.status = 404;
const url = ctx.get('x-request-url');
if (url !== '') {
$('#content').text(url + ' not found');
}
}
ctx.vary('Accept-Language');
ctx.type = 'text/html';
ctx.body = $.html();
}
export async function renderGroup(ctx) {
const headers = ctxHeaders(ctx);
const _gid = ctx.params.id;
const gid = ctx.params.gid;
const $ = cheerio.load(groupTpl);
const lang = headers['x-language'] ?? 'eng';
const info = metaInfos[lang];
if (info) {
$('title').text(info.title);
$('meta[name="description"]').prop('content', info.desc);
const siteInfo = metaInfos[lang];
if (siteInfo) {
$('title').text(siteInfo.title);
$('meta[name="description"]').prop('content', siteInfo.desc);
}
try {
const group = await getGroup(headers, _gid);
const gid = Xid.fromValue(group.id);
const group = await getGroup(headers, gid);
const xGid = Xid.fromValue(group.id);
const groupUrl = `${siteBase}/group/${gid.toString()}`;
$('meta[property="og:url"]').prop('content', groupUrl);
$('meta[property="og:title"]').prop('content', group.name);
$('meta[property="og:description"]').prop('content', group.slogan);
$('meta[property="og:url"]').prop('content', groupUrl);
$('#group_name').text(group.name);
$('#group_slogan').text(group.slogan);
await Promise.all([
(async () => {
const docs = await listCollections(headers, gid);
const docs = await listCollections(headers, xGid);
renderCollectionItems($, docs, lang);
})().catch(ignoreError),
(async () => {
const docs = await listPublications(headers, gid);
const docs = await listPublications(headers, xGid);
renderPublicationItems($, docs);
})().catch(ignoreError),
]);
Expand Down Expand Up @@ -199,6 +274,25 @@ function renderCollectionItems($, docs, lang) {
})
.filter((item) => !!item));
}
function renderCollectionChildrenItems($, docs) {
renderList($, 'children', docs
.map((doc) => {
if (doc.kind == 2)
return null; // collection
const cid = Xid.fromValue(doc.cid).toString();
const gid = Xid.fromValue(doc.gid).toString();
return {
id: `${gid}-${cid}`,
url: `${siteBase}/pub/${cid}?gid=${gid}`,
title: doc.title,
language: doc.language,
summary: doc.summary ?? '',
keywords: doc.keywords,
authors: doc.authors,
};
})
.filter((item) => !!item));
}
function renderList($, ulId, items) {
const ul = $('#' + ulId);
for (const item of items) {
Expand Down Expand Up @@ -339,6 +433,22 @@ async function listLatestPublications(headers) {
const obj = decode(Buffer.from(data));
return obj.result;
}
async function getCollection(headers, cid) {
const api = new URL('/v1/collection', writingBase);
api.searchParams.append('gid', '000000000000000anon0');
api.searchParams.append('id', cid);
api.searchParams.append('fields', 'info,updated_at');
headers.accept = 'application/cbor';
const res = await fetch(api, {
headers,
});
if (res.status !== 200) {
throw createError(res.status, await res.text());
}
const data = await res.arrayBuffer();
const obj = decode(Buffer.from(data));
return obj.result;
}
async function listCollections(headers, gid) {
const api = new URL('/v1/collection/list', writingBase);
headers.accept = 'application/cbor';
Expand Down Expand Up @@ -378,6 +488,26 @@ async function listLatestCollections(headers) {
const obj = decode(Buffer.from(data));
return obj.result;
}
async function listCollectionChildren(headers, id) {
const api = new URL('/v1/collection/list_children', writingBase);
headers.accept = 'application/cbor';
headers['content-type'] = 'application/cbor';
const res = await fetch(api, {
method: 'POST',
headers,
body: Buffer.from(encode({
gid: Xid.default().toBytes(),
id: id.toBytes(),
page_size: 100,
})),
});
if (res.status !== 200) {
throw createError(res.status, await res.text());
}
const data = await res.arrayBuffer();
const obj = decode(Buffer.from(data));
return obj.result;
}
function isXid(id) {
try {
Xid.parse(id);
Expand Down
34 changes: 34 additions & 0 deletions html/collection.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
<!DOCTYPE html>
<html lang="eng">

<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="https://cdn.yiwen.pub/web/logo.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Yiwen — AI-based Translingual Knowledge Content Platform</title>
<meta name="description"
content="Yiwen AI is a multilingual knowledge-sharing platform. With the help of ChatGPT, you can effortlessly translate and share captivating articles and documents with readers worldwide, breaking down language barriers in knowledge acquisition." />
<meta property="og:url" content="https://www.yiwen.ai/" />
<meta property="og:site_name" content="Yiwen — AI-based Translingual Knowledge Content Platform" />
<meta property="og:type" content="website" />
<meta property="og:title" content="" />
<meta property="og:description" content="" />
</head>

<body>
<div id="root">
<h1 id="title" title="title"></h1>
<div id="summary" title="summary"></div>
<div id="authors" title="authors"></div>
<div id="keywords" title="authors"></div>
<div>
<a id="group" title="group" href="" target="_blank"></a>
<span id="updated_time"></span>
</div>
<div title="children">
<ul id="children"></ul>
</div>
</div>
</body>

</html>
2 changes: 2 additions & 0 deletions html/group.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
<meta property="og:url" content="https://www.yiwen.ai/" />
<meta property="og:site_name" content="Yiwen — AI-based Translingual Knowledge Content Platform" />
<meta property="og:type" content="website" />
<meta property="og:title" content="" />
<meta property="og:description" content="" />
</head>

<body>
Expand Down
11 changes: 8 additions & 3 deletions html/publication.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,21 @@
<meta property="og:url" content="https://www.yiwen.ai/" />
<meta property="og:site_name" content="Yiwen — AI-based Translingual Knowledge Content Platform" />
<meta property="og:type" content="website" />
<meta property="og:title" content="" />
<meta property="og:description" content="" />
</head>

<body>
<div id="root">
<div>
<div title="other languages version">
<ul id="publications"></ul>
</div>
<h1 id="title"></h1>
<h1 id="title" title="title"></h1>
<div id="summary" title="summary"></div>
<div id="authors" title="authors"></div>
<div id="keywords" title="authors"></div>
<div>
<a id="authors" href="" target="_blank"></a>
<a id="group" title="group" href="" target="_blank"></a>
<span id="updated_time"></span>
<span id="version"></span>
</div>
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "webscraper",
"version": "1.2.0",
"version": "1.2.1",
"description": "",
"private": true,
"main": "dist/main.js",
Expand Down
Loading

0 comments on commit af5379c

Please sign in to comment.