From 896c6121019b046a882e208651787d2f6f978d8b Mon Sep 17 00:00:00 2001 From: 0xZensh Date: Sun, 1 Oct 2023 16:45:58 +0800 Subject: [PATCH] feat: add SSR for robots --- .eslintrc.cjs | 52 ++++---- Dockerfile | 1 + config/default.json | 5 +- dist/app.js | 16 ++- dist/ssr.js | 207 +++++++++++++++++++++++++++++ dist/tiptap.js | 122 ++++++----------- html/group.html | 29 +++++ html/index.html | 25 ++++ html/publication.html | 29 +++++ package.json | 4 +- src/app.ts | 17 ++- src/ssr.ts | 297 ++++++++++++++++++++++++++++++++++++++++++ tsconfig.json | 4 +- 13 files changed, 690 insertions(+), 118 deletions(-) create mode 100644 dist/ssr.js create mode 100644 html/group.html create mode 100644 html/index.html create mode 100644 html/publication.html create mode 100644 src/ssr.ts diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 2ab21d3..ca68adc 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -1,29 +1,29 @@ /* eslint-env node */ module.exports = { - env: { - browser: true, - es2021: true - }, - extends: [ - "plugin:@typescript-eslint/recommended-type-checked", - "plugin:@typescript-eslint/stylistic-type-checked" - ], - plugins: ["@typescript-eslint"], - parser: "@typescript-eslint/parser", - parserOptions: { - project: ["./tsconfig.eslint.json"], - ecmaVersion: "latest", - sourceType: "module" - }, - rules: { - // Note: you must disable the base rule as it can report incorrect errors - "space-before-function-paren": "off", - "@typescript-eslint/space-before-function-paren": "off", - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-unsafe-assignment": "off", - "@typescript-eslint/no-unsafe-return": "off", - "@typescript-eslint/no-unsafe-member-access": "off", - "@typescript-eslint/no-unsafe-argument": "off", - }, - root: true, + env: { + browser: true, + es2022: true, + }, + extends: [ + 'plugin:@typescript-eslint/recommended-type-checked', + 'plugin:@typescript-eslint/stylistic-type-checked', + ], + plugins: ['@typescript-eslint'], + parser: '@typescript-eslint/parser', + parserOptions: { + project: ['./tsconfig.eslint.json'], + ecmaVersion: 'latest', + sourceType: 'module', + }, + rules: { + // Note: you must disable the base rule as it can report incorrect errors + 'space-before-function-paren': 'off', + '@typescript-eslint/space-before-function-paren': 'off', + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unsafe-assignment': 'off', + '@typescript-eslint/no-unsafe-return': 'off', + '@typescript-eslint/no-unsafe-member-access': 'off', + '@typescript-eslint/no-unsafe-argument': 'off', + }, + root: true, } diff --git a/Dockerfile b/Dockerfile index 3be4202..fefd658 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN pnpm install --prod \ # for most source file changes. COPY --chown=myuser dist ./dist/ COPY --chown=myuser config ./config/ +COPY --chown=myuser html ./html/ ENV NODE_ENV=production # Run the image. diff --git a/config/default.json b/config/default.json index 4efceb0..700426f 100644 --- a/config/default.json +++ b/config/default.json @@ -9,5 +9,8 @@ "contactPoints": [ "127.0.0.1:9042" ] - } + }, + "siteBase": "http://127.0.0.1:8080", + "writingBase": "http://127.0.0.1:8080", + "userBase": "http://127.0.0.1:8080" } \ No newline at end of file diff --git a/dist/app.js b/dist/app.js index 5899660..4a7d96a 100644 --- a/dist/app.js +++ b/dist/app.js @@ -3,7 +3,8 @@ import Router from '@koa/router'; import { encode } from 'cborg'; import { LogLevel, createLog, writeLog } from './log.js'; import { connect } from './db/scylladb.js'; -import { versionAPI, healthzAPI, scrapingAPI, searchAPI, documentAPI, convertingAPI, } from './api.js'; +import { healthzAPI, scrapingAPI, searchAPI, documentAPI, convertingAPI, } from './api.js'; +import { renderIndex, renderPublication, renderGroup } from './ssr.js'; const GZIP_MIN_LENGTH = 128; export async function initApp(app) { // attach stateful components to the application context @@ -11,12 +12,14 @@ export async function initApp(app) { // create routes const router = new Router(); router.use(initContext); - router.get('/', versionAPI); + router.get('/', renderIndex); router.get('/healthz', healthzAPI); router.get('/v1/scraping', scrapingAPI); router.get('/v1/search', searchAPI); router.get('/v1/document', documentAPI); router.post('/v1/converting', convertingAPI); + router.get('/pub/:id', renderPublication); + router.get('/group/:id', renderGroup); app.use(router.routes()); app.use(router.allowedMethods()); } @@ -99,4 +102,13 @@ async function initContext(ctx, next) { } ctx.body = body; } + else if (typeof body === 'string') { + if (body.length > GZIP_MIN_LENGTH && + ctx.acceptsEncodings('gzip') === 'gzip') { + log.beforeGzip = body.length; + ctx.body = gzipSync(Buffer.from(body, 'utf-8')); + ctx.remove('Content-Length'); + ctx.set('content-encoding', 'gzip'); + } + } } diff --git a/dist/ssr.js b/dist/ssr.js new file mode 100644 index 0000000..7d6b85f --- /dev/null +++ b/dist/ssr.js @@ -0,0 +1,207 @@ +import { readFileSync } from 'node:fs'; +import { URL } from 'node:url'; +import config from 'config'; +import { decode, encode } from 'cborg'; +import { Xid } from 'xid-ts'; +import * as cheerio from 'cheerio'; +import createError from 'http-errors'; +import { toHTML } from './tiptap.js'; +const indexTpl = readFileSync('./html/index.html', 'utf-8'); +const publicationTpl = readFileSync('./html/publication.html', 'utf-8'); +const groupTpl = readFileSync('./html/group.html', 'utf-8'); +const siteBase = config.get('siteBase'); +const writingBase = config.get('writingBase'); +const userBase = config.get('userBase'); +export async function renderIndex(ctx) { + const ctxheaders = { + 'x-request-id': ctx.get('x-request-id'), + 'x-auth-user': '000000000000000anon0', + 'x-auth-user-rating': ctx.get('x-auth-user-rating'), + 'x-auth-app': ctx.get('x-auth-app'), + 'x-language': ctx.get('x-language'), + }; + const $ = cheerio.load(indexTpl); + try { + const docs = await listIndex(ctxheaders); + for (const doc of docs) { + const cid = Xid.fromValue(doc.cid).toString(); + const docUrl = `${siteBase}/pub/${cid}?gid=${Xid.fromValue(doc.gid).toString()}`; + $('ul').append(`
    • `); + $(`#${cid}`).text(doc.title); + } + } + catch (err) { + ctx.status = 404; + const url = ctx.get('x-request-url'); + if (url !== '') { + $('#content').text(url + ' not found'); + } + } + ctx.vary('Accept-Language'); + ctx.type = 'text/html'; + ctx.body = $.html(); +} +export async function renderPublication(ctx) { + const ctxheaders = { + 'x-request-id': ctx.get('x-request-id'), + 'x-auth-user': '000000000000000anon0', + 'x-auth-user-rating': ctx.get('x-auth-user-rating'), + 'x-auth-app': ctx.get('x-auth-app'), + 'x-language': ctx.get('x-language'), + }; + const cid = ctx.params.id; + const { gid, language } = ctx.query; + const $ = cheerio.load(publicationTpl); + try { + const doc = await getPublication(ctxheaders, cid, (gid ?? ''), (language ?? '')); + const docUrl = `${siteBase}/pub/${Xid.fromValue(doc.cid).toString()}`; + const groupUrl = `${siteBase}/group/${Xid.fromValue(doc.gid).toString()}`; + $('html').prop('lang', doc.language); + $('meta[property="og:title"]').prop('content', doc.title); + $('meta[property="og:url"]').prop('content', docUrl); + $('#title').text(doc.title); + const authors = $('#authors'); + authors.prop('href', groupUrl); + authors.text(groupUrl); + if (doc.authors != null && doc.authors.length > 0) { + authors.text(doc.authors.join(', ')); + } + const updated_at = new Date(doc.updated_at).toUTCString(); + $('#updated_time').text(updated_at); + $('#version').text(doc.version.toString()); + const content = decode(doc.content); + $('#content').html(toHTML(content) + + `\n

    ${docUrl}

    `); + ctx.set('last-modified', updated_at); + } + catch (err) { + ctx.status = 404; + const url = ctx.get('x-request-url'); + if (url !== '') { + $('#content').text(url + ' not found'); + } + } + ctx.vary('Accept-Language'); + ctx.type = 'text/html'; + ctx.body = $.html(); +} +export async function renderGroup(ctx) { + const ctxheaders = { + 'x-request-id': ctx.get('x-request-id'), + 'x-auth-user': '000000000000000anon0', + 'x-auth-user-rating': ctx.get('x-auth-user-rating'), + 'x-auth-app': ctx.get('x-auth-app'), + 'x-language': ctx.get('x-language'), + }; + const gid = ctx.params.id; + const $ = cheerio.load(groupTpl); + try { + const group = await getGroup(ctxheaders, gid); + const groupUrl = `${siteBase}/group/${Xid.fromValue(group.id).toString()}`; + $('meta[property="og:title"]').prop('content', group.name); + $('meta[property="og:description"]').prop('content', group.slogan); + $('meta[property="og:url"]').prop('content', groupUrl); + $('#group_name').text(group.name); + $('#group_slogan').text(group.slogan); + const docs = await listPublications(ctxheaders, Xid.fromValue(group.id)); + for (const doc of docs) { + const cid = Xid.fromValue(doc.cid).toString(); + const docUrl = `${siteBase}/pub/${cid}?gid=${Xid.fromValue(doc.gid).toString()}`; + $('ul').append(`
    • `); + $(`#${cid}`).text(doc.title); + } + } + catch (err) { + ctx.status = 404; + const url = ctx.get('x-request-url'); + if (url !== '') { + $('#content').text(url + ' not found'); + } + } + ctx.vary('Accept-Language'); + ctx.type = 'text/html'; + ctx.body = $.html(); +} +async function getGroup(headers, gid) { + const api = new URL('/v1/group', userBase); + if (isXid(gid)) { + api.searchParams.append('id', gid); + } + else { + api.searchParams.append('cn', gid); + } + api.searchParams.append('fields', 'cn,name,status,slogan'); + headers.accept = 'application/cbor'; + const res = await fetch(api, { + headers, + }); + if (res.status !== 200) { + throw createError(res.status, await res.text()); + } + const data = await res.arrayBuffer(); + const obj = decode(Buffer.from(data)); + return obj.result; +} +async function getPublication(headers, cid, gid, language) { + const api = new URL('/v1/publication/implicit_get', writingBase); + api.searchParams.append('cid', cid); + if (gid !== '') { + api.searchParams.append('gid', gid); + } + if (language !== '') { + api.searchParams.append('language', language); + } + api.searchParams.append('fields', 'title,updated_at,authors,content'); + headers.accept = 'application/cbor'; + const res = await fetch(api, { + headers, + }); + if (res.status !== 200) { + throw createError(res.status, await res.text()); + } + const data = await res.arrayBuffer(); + const obj = decode(Buffer.from(data)); + return obj.result; +} +async function listPublications(headers, gid) { + const api = new URL('/v1/publication/list', writingBase); + headers.accept = 'application/cbor'; + headers['content-type'] = 'application/cbor'; + const res = await fetch(api, { + method: 'POST', + headers, + body: Buffer.from(encode({ + gid: gid.toBytes(), + status: 2, + fields: ['title', 'updated_at'], + })), + }); + if (res.status !== 200) { + throw createError(res.status, await res.text()); + } + const data = await res.arrayBuffer(); + const obj = decode(Buffer.from(data)); + return obj.result; +} +async function listIndex(headers) { + const api = new URL('/v1/search?q=', writingBase); + headers.accept = 'application/cbor'; + headers['content-type'] = 'application/cbor'; + const res = await fetch(api, { + headers, + }); + if (res.status !== 200) { + throw createError(res.status, await res.text()); + } + const data = await res.arrayBuffer(); + const obj = decode(Buffer.from(data)); + return obj.result.hits; +} +function isXid(id) { + try { + Xid.parse(id); + return true; + } + catch (e) { } + return false; +} diff --git a/dist/tiptap.js b/dist/tiptap.js index fdae68d..5caeed7 100644 --- a/dist/tiptap.js +++ b/dist/tiptap.js @@ -2,37 +2,24 @@ import { nanoid } from 'nanoid'; // import { generateJSON, generateHTML } from '@tiptap/html' import { generateJSON, generateHTML } from './html.js'; -import Color from '@tiptap/extension-color'; -import Bold from '@tiptap/extension-bold'; -import Document from '@tiptap/extension-document'; -import Blockquote from '@tiptap/extension-blockquote'; -import Code from '@tiptap/extension-code'; -import CodeBlock from '@tiptap/extension-code-block'; -// import FontFamily from '@tiptap/extension-font-family' -import HardBreak from '@tiptap/extension-hard-break'; -import Heading from '@tiptap/extension-heading'; -import HorizontalRule from '@tiptap/extension-horizontal-rule'; -import Image from '@tiptap/extension-image'; -import Italic from '@tiptap/extension-italic'; -import Link from '@tiptap/extension-link'; -import ListItem from '@tiptap/extension-list-item'; -import Mention from '@tiptap/extension-mention'; -import OrderedList from '@tiptap/extension-ordered-list'; -import Paragraph from '@tiptap/extension-paragraph'; -import Subscript from '@tiptap/extension-subscript'; -import Superscript from '@tiptap/extension-superscript'; -import Table from '@tiptap/extension-table'; -import TableCell from '@tiptap/extension-table-cell'; -import TableHeader from '@tiptap/extension-table-header'; -import TableRow from '@tiptap/extension-table-row'; -import TaskItem from '@tiptap/extension-task-item'; -import TaskList from '@tiptap/extension-task-list'; -import Text from '@tiptap/extension-text'; -import TextAlign from '@tiptap/extension-text-align'; -import TextStyle from '@tiptap/extension-text-style'; -import Typography from '@tiptap/extension-typography'; -import Underline from '@tiptap/extension-underline'; -import Youtube from '@tiptap/extension-youtube'; +import { StarterKit } from '@tiptap/starter-kit'; +import { Color } from '@tiptap/extension-color'; +import { Image } from '@tiptap/extension-image'; +import { Link } from '@tiptap/extension-link'; +import { Mention } from '@tiptap/extension-mention'; +import { Subscript } from '@tiptap/extension-subscript'; +import { Superscript } from '@tiptap/extension-superscript'; +import { Table } from '@tiptap/extension-table'; +import { TableCell } from '@tiptap/extension-table-cell'; +import { TableHeader } from '@tiptap/extension-table-header'; +import { TableRow } from '@tiptap/extension-table-row'; +import { TaskItem } from '@tiptap/extension-task-item'; +import { TaskList } from '@tiptap/extension-task-list'; +import { TextAlign } from '@tiptap/extension-text-align'; +import { TextStyle } from '@tiptap/extension-text-style'; +import { Typography } from '@tiptap/extension-typography'; +import { Underline } from '@tiptap/extension-underline'; +import { Youtube } from '@tiptap/extension-youtube'; import { Details } from '@tiptap-pro/extension-details'; import { DetailsSummary } from '@tiptap-pro/extension-details-summary'; import { DetailsContent } from '@tiptap-pro/extension-details-content'; @@ -50,9 +37,9 @@ const uidTypes = [ 'paragraph', 'tableHeader', 'tableCell', + 'taskItem', ]; const tiptapExtensions = [ - Document, Details.configure({ persist: true, }), @@ -63,31 +50,20 @@ const tiptapExtensions = [ emojis: [...emojis], }), Color, - Bold, - Blockquote, - Code, - CodeBlock, - // FontFamily, - HardBreak, - Heading, - HorizontalRule, Image, - Italic, Link.configure({ - openOnClick: false, - linkOnPaste: false, + protocols: [], autolink: false, - validate: isValidHref, - HTMLAttributes: { - rel: '', - target: '', - }, + linkOnPaste: true, + openOnClick: false, + HTMLAttributes: { target: '_blank', rel: 'noopener noreferrer' }, + validate: (href) => (href ? href.startsWith('https://') : false), }), - ListItem, - Mathematics, + Mathematics.configure({ katexOptions: { strict: false } }), Mention, - OrderedList, - Paragraph, + StarterKit.configure({ + heading: { levels: [1, 2, 3, 4, 5, 6] }, + }), Subscript, Superscript, Table, @@ -98,8 +74,16 @@ const tiptapExtensions = [ nested: true, }), TaskList, - Text, - TextAlign, + TextAlign.configure({ + types: [ + 'heading', + 'paragraph', + 'codeBlock', + 'blockquote', + 'table', + 'tableCell', + ], + }), TextStyle, Typography, Underline, @@ -145,13 +129,8 @@ export class JSONDocumentAmender { for (const mark of node.marks) { if (mark.type === 'link' && mark.attrs != null) { delete mark.attrs.class; - if (isSameOriginHref(mark.attrs.href)) { - delete mark.attrs.target; - } - else { - mark.attrs.rel = 'noopener noreferrer'; - mark.attrs.target = '_blank'; - } + mark.attrs.rel = 'noopener noreferrer'; + mark.attrs.target = '_blank'; } } } @@ -194,24 +173,3 @@ export function findTitle(doc, level) { } return ''; } -const LOCALHOST = 'https://localhost'; -function isSameOriginHref(href) { - if (typeof href === 'string') { - try { - const url = new URL(href, LOCALHOST); - return url.origin === LOCALHOST; - } - catch (e) { } - } - return false; -} -function isValidHref(href) { - if (typeof href === 'string') { - try { - const url = new URL(href, LOCALHOST); - return url.protocol === 'https:' || url.protocol === 'mailto:'; - } - catch (e) { } - } - return false; -} diff --git a/html/group.html b/html/group.html new file mode 100644 index 0000000..cad8f92 --- /dev/null +++ b/html/group.html @@ -0,0 +1,29 @@ + + + + + + + + Yiwen AI + + + + + + + + + +
    +
    + + +
    +
    +
      +
      +
      + + + \ No newline at end of file diff --git a/html/index.html b/html/index.html new file mode 100644 index 0000000..c096944 --- /dev/null +++ b/html/index.html @@ -0,0 +1,25 @@ + + + + + + + + Yiwen AI + + + + + + + + + +
      +
      +
        +
        +
        + + + \ No newline at end of file diff --git a/html/publication.html b/html/publication.html new file mode 100644 index 0000000..c225c45 --- /dev/null +++ b/html/publication.html @@ -0,0 +1,29 @@ + + + + + + + + Yiwen AI + + + + + + + + + +
        +

        +
        + + + +
        +
        Not found
        +
        + + + \ No newline at end of file diff --git a/package.json b/package.json index 76be51e..595d905 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webscraper", - "version": "1.0.0", + "version": "1.1.0", "description": "", "private": true, "main": "dist/main.js", @@ -8,7 +8,7 @@ "scripts": { "start": "npm run start:dev", "start:prod": "node dist/main.js", - "start:dev": "ts-node-esm -T src/main.ts", + "start:dev": "ts-node --esm src/main.ts", "debug": "ts-node-esm -T debug/format.ts", "build": "rm -rf dist && tsc", "lint": "eslint .", diff --git a/src/app.ts b/src/app.ts index f826f3e..dd816b9 100644 --- a/src/app.ts +++ b/src/app.ts @@ -6,7 +6,6 @@ import { encode } from 'cborg' import { LogLevel, createLog, writeLog } from './log.js' import { connect } from './db/scylladb.js' import { - versionAPI, healthzAPI, scrapingAPI, searchAPI, @@ -14,6 +13,8 @@ import { convertingAPI, } from './api.js' +import { renderIndex, renderPublication, renderGroup } from './ssr.js' + const GZIP_MIN_LENGTH = 128 export async function initApp(app: Koa): Promise { @@ -23,12 +24,14 @@ export async function initApp(app: Koa): Promise { // create routes const router = new Router() router.use(initContext) - router.get('/', versionAPI) + router.get('/', renderIndex) router.get('/healthz', healthzAPI) router.get('/v1/scraping', scrapingAPI) router.get('/v1/search', searchAPI) router.get('/v1/document', documentAPI) router.post('/v1/converting', convertingAPI) + router.get('/pub/:id', renderPublication) + router.get('/group/:id', renderGroup) app.use(router.routes()) app.use(router.allowedMethods()) @@ -126,5 +129,15 @@ async function initContext(ctx: Koa.Context, next: Koa.Next): Promise { } ctx.body = body + } else if (typeof body === 'string') { + if ( + body.length > GZIP_MIN_LENGTH && + ctx.acceptsEncodings('gzip') === 'gzip' + ) { + log.beforeGzip = body.length + ctx.body = gzipSync(Buffer.from(body, 'utf-8')) + ctx.remove('Content-Length') + ctx.set('content-encoding', 'gzip') + } } } diff --git a/src/ssr.ts b/src/ssr.ts new file mode 100644 index 0000000..370f9f5 --- /dev/null +++ b/src/ssr.ts @@ -0,0 +1,297 @@ +import { readFileSync } from 'node:fs' +import { URL } from 'node:url' +import { type Context } from 'koa' +import config from 'config' +import { decode, encode } from 'cborg' +import { Xid } from 'xid-ts' +import * as cheerio from 'cheerio' +import createError from 'http-errors' +import { toHTML, type Node } from './tiptap.js' + +const indexTpl = readFileSync('./html/index.html', 'utf-8') +const publicationTpl = readFileSync('./html/publication.html', 'utf-8') +const groupTpl = readFileSync('./html/group.html', 'utf-8') +const siteBase = config.get('siteBase') +const writingBase = config.get('writingBase') +const userBase = config.get('userBase') + +export async function renderIndex(ctx: Context) { + const ctxheaders: Record = { + 'x-request-id': ctx.get('x-request-id'), + 'x-auth-user': '000000000000000anon0', + 'x-auth-user-rating': ctx.get('x-auth-user-rating'), + 'x-auth-app': ctx.get('x-auth-app'), + 'x-language': ctx.get('x-language'), + } + + const $ = cheerio.load(indexTpl) + + try { + const docs = await listIndex(ctxheaders) + for (const doc of docs) { + const cid = Xid.fromValue(doc.cid).toString() + const docUrl = `${siteBase}/pub/${cid}?gid=${Xid.fromValue( + doc.gid + ).toString()}` + $('ul').append( + `
        • ` + ) + $(`#${cid}`).text(doc.title) + } + } catch (err: any) { + ctx.status = 404 + const url = ctx.get('x-request-url') + if (url !== '') { + $('#content').text(url + ' not found') + } + } + + ctx.vary('Accept-Language') + ctx.type = 'text/html' + ctx.body = $.html() +} + +export async function renderPublication(ctx: Context): Promise { + const ctxheaders: Record = { + 'x-request-id': ctx.get('x-request-id'), + 'x-auth-user': '000000000000000anon0', + 'x-auth-user-rating': ctx.get('x-auth-user-rating'), + 'x-auth-app': ctx.get('x-auth-app'), + 'x-language': ctx.get('x-language'), + } + + const cid = ctx.params.id as string + const { gid, language } = ctx.query + const $ = cheerio.load(publicationTpl) + + try { + const doc = await getPublication( + ctxheaders, + cid, + (gid ?? '') as string, + (language ?? '') as string + ) + + const docUrl = `${siteBase}/pub/${Xid.fromValue(doc.cid).toString()}` + const groupUrl = `${siteBase}/group/${Xid.fromValue(doc.gid).toString()}` + $('html').prop('lang', doc.language) + $('meta[property="og:title"]').prop('content', doc.title) + $('meta[property="og:url"]').prop('content', docUrl) + + $('#title').text(doc.title) + const authors = $('#authors') + authors.prop('href', groupUrl) + authors.text(groupUrl) + if (doc.authors != null && doc.authors.length > 0) { + authors.text(doc.authors.join(', ')) + } + + const updated_at = new Date(doc.updated_at).toUTCString() + $('#updated_time').text(updated_at) + $('#version').text(doc.version.toString()) + + const content = decode(doc.content) as Node + $('#content').html( + toHTML(content) + + `\n

        ${docUrl}

        ` + ) + + ctx.set('last-modified', updated_at) + } catch (err: any) { + ctx.status = 404 + const url = ctx.get('x-request-url') + if (url !== '') { + $('#content').text(url + ' not found') + } + } + + ctx.vary('Accept-Language') + ctx.type = 'text/html' + ctx.body = $.html() +} + +export async function renderGroup(ctx: Context) { + const ctxheaders: Record = { + 'x-request-id': ctx.get('x-request-id'), + 'x-auth-user': '000000000000000anon0', + 'x-auth-user-rating': ctx.get('x-auth-user-rating'), + 'x-auth-app': ctx.get('x-auth-app'), + 'x-language': ctx.get('x-language'), + } + + const gid = ctx.params.id as string + const $ = cheerio.load(groupTpl) + + try { + const group = await getGroup(ctxheaders, gid) + const groupUrl = `${siteBase}/group/${Xid.fromValue(group.id).toString()}` + $('meta[property="og:title"]').prop('content', group.name) + $('meta[property="og:description"]').prop('content', group.slogan) + $('meta[property="og:url"]').prop('content', groupUrl) + + $('#group_name').text(group.name) + $('#group_slogan').text(group.slogan) + + const docs = await listPublications(ctxheaders, Xid.fromValue(group.id)) + for (const doc of docs) { + const cid = Xid.fromValue(doc.cid).toString() + const docUrl = `${siteBase}/pub/${cid}?gid=${Xid.fromValue( + doc.gid + ).toString()}` + $('ul').append( + `
        • ` + ) + $(`#${cid}`).text(doc.title) + } + } catch (err: any) { + ctx.status = 404 + const url = ctx.get('x-request-url') + if (url !== '') { + $('#content').text(url + ' not found') + } + } + + ctx.vary('Accept-Language') + ctx.type = 'text/html' + ctx.body = $.html() +} + +interface GroupInfo { + id: Uint8Array + cn: string + name: string + logo: string + slogan: string + status: number +} + +async function getGroup( + headers: Record, + gid: string +): Promise { + const api = new URL('/v1/group', userBase) + if (isXid(gid)) { + api.searchParams.append('id', gid) + } else { + api.searchParams.append('cn', gid) + } + + api.searchParams.append('fields', 'cn,name,status,slogan') + + headers.accept = 'application/cbor' + const res = await fetch(api, { + headers, + }) + + if (res.status !== 200) { + throw createError(res.status, await res.text()) + } + + const data = await res.arrayBuffer() + const obj = decode(Buffer.from(data)) + return obj.result +} + +interface PublicationOutput { + gid: Uint8Array + cid: Uint8Array + language: string + version: number + rating?: number + status: number + created_at: number + updated_at: number + model: string + original_url?: string + from_language?: string + title: string + cover?: string + authors?: string[] + summary?: string + content: Uint8Array +} + +async function getPublication( + headers: Record, + cid: string, + gid: string, + language: string +): Promise { + const api = new URL('/v1/publication/implicit_get', writingBase) + api.searchParams.append('cid', cid) + if (gid !== '') { + api.searchParams.append('gid', gid) + } + if (language !== '') { + api.searchParams.append('language', language) + } + api.searchParams.append('fields', 'title,updated_at,authors,content') + + headers.accept = 'application/cbor' + const res = await fetch(api, { + headers, + }) + + if (res.status !== 200) { + throw createError(res.status, await res.text()) + } + + const data = await res.arrayBuffer() + const obj = decode(Buffer.from(data)) + return obj.result +} + +async function listPublications( + headers: Record, + gid: Xid +): Promise { + const api = new URL('/v1/publication/list', writingBase) + headers.accept = 'application/cbor' + headers['content-type'] = 'application/cbor' + const res = await fetch(api, { + method: 'POST', + headers, + body: Buffer.from( + encode({ + gid: gid.toBytes(), + status: 2, + fields: ['title', 'updated_at'], + }) + ), + }) + + if (res.status !== 200) { + throw createError(res.status, await res.text()) + } + + const data = await res.arrayBuffer() + const obj = decode(Buffer.from(data)) + return obj.result +} + +async function listIndex( + headers: Record +): Promise { + const api = new URL('/v1/search?q=', writingBase) + headers.accept = 'application/cbor' + headers['content-type'] = 'application/cbor' + const res = await fetch(api, { + headers, + }) + + if (res.status !== 200) { + throw createError(res.status, await res.text()) + } + + const data = await res.arrayBuffer() + const obj = decode(Buffer.from(data)) + return obj.result.hits +} + +function isXid(id: string): boolean { + try { + Xid.parse(id) + return true + } catch (e) {} + return false +} diff --git a/tsconfig.json b/tsconfig.json index ceab198..0bb5dcb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,7 +11,5 @@ "skipLibCheck": true, "strict": true }, - "include": [ - "src/**/*.ts" - ] + "include": ["src/**/*.ts"] }