diff --git a/.github/workflows/build-synapse.yml b/.github/workflows/build-synapse.yml new file mode 100644 index 0000000..06c891c --- /dev/null +++ b/.github/workflows/build-synapse.yml @@ -0,0 +1,30 @@ +on: + push: + branches: + - main + paths: + - "!.github/**" + - .github/workflows/build-synapse.yml + - src/** + pull_request: + branches: + - main + paths: + - "!.github/**" + - .github/workflows/build-synapse.yml + - src/** + +jobs: + run_test: + runs-on: + - ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v3 + - run: curl -fsSL https://synap.sh/install | bash + - run: echo "SYNAPSE_INSTALL=$HOME/.synapse" >> "$GITHUB_ENV" + - run: echo "${SYNAPSE_INSTALL}/bin" >> "$GITHUB_PATH" + - run: synapse --version + - run: synapse compile && synapse build + - run: ./dist/bin/synapse diff --git a/.gitignore b/.gitignore index acdbaf9..8068005 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ node_modules **/.env **/.env.* **/.DS_Store +*.d.zig.ts \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md index 3dd95a8..39d1107 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,3 +1,30 @@ +## Prerequisites +You'll need the latest version of Synapse installed + +## Synapse CLI +The bulk of Synapse lives in `src`. Use `synapse compile` in the root of the repository to build. There is a fast and slow way to test your changes. The fast way requires a little bit of setup but results in much quicker iterations. The slow way creates an executable which is most similar to the release build of Synapse. + +### Slow Way + +Run `synapse compile` + `synapse build`. The executable will be placed in `dist/bin` e.g. `dist/bin/synapse` for Linux/macOS and `dist/bin/synapse.exe` for Windows. + +### Fast Way + +This is currently unreliable. Use the slow way for now. + + + + ## Integrations (aka compiler backends) Synapse uses a plugin-like architecture for loading deployment target implementations. Packages can contribute implementations by using the `addTarget` function from `synapse:core`. See [this file](../integrations/local/src/function.ts) for a reasonably simple example. @@ -20,3 +47,10 @@ This will use the `local` target by default. You can change the target by adding cd test/conformance && synapse compile --target aws && synapse test ``` +## The `packages` directory + +The tarballs in this directory contain services (or rather, service stubs) that may eventually be used in either Synapse directly, or possibly a separate CLI entirely. These are currently closed-source for two main reasons: + +1. It's difficult to open-source _live_ services (but Synapse _will_ make it easier!) +2. They may become apart of a strategy to help fund the development of Synapse + diff --git a/package.json b/package.json new file mode 100644 index 0000000..94ddfd5 --- /dev/null +++ b/package.json @@ -0,0 +1,42 @@ +{ + "name": "synapse", + "version": "0.0.3", + "bin": "./src/cli/index.ts", + "dependencies": { + "esbuild": "^0.20.2", + "typescript": "~5.4.5" + }, + "devDependencies": { + "@types/node": "^20.11.27", + "postject": "github:Cohesible/postject", + "@cohesible/auth": "file:packages/auth.tgz", + "@cohesible/quotes": "file:packages/quotes.tgz", + "@cohesible/resources": "file:packages/resources.tgz" + }, + "engines": { + "node": "22.1.0" + }, + "scripts": { + "compileSelf": "synapse compile --no-synth && synapse publish --local" + }, + "files": [ + "dist" + ], + "synapse": { + "config": { + "target": "local", + "zigFiles": [ + "src/zig/ast.zig", + "src/zig/fs-ext.zig", + "src/zig/util.zig" + ] + }, + "binaryDependencies": { + "node": "https://github.com/Cohesible/node.git", + "terraform": "https://github.com/Cohesible/terraform.git" + }, + "devTools": { + "zig": "0.12.0" + } + } +} \ No newline at end of file diff --git a/packages/auth.tgz b/packages/auth.tgz new file mode 100644 index 0000000..19d1a57 Binary files /dev/null and b/packages/auth.tgz differ diff --git a/packages/quotes.tgz b/packages/quotes.tgz new file mode 100644 index 0000000..919cb37 Binary files /dev/null and b/packages/quotes.tgz differ diff --git a/packages/resources.tgz b/packages/resources.tgz new file mode 100644 index 0000000..65189d0 Binary files /dev/null and b/packages/resources.tgz differ diff --git a/src/artifacts.ts b/src/artifacts.ts new file mode 100644 index 0000000..90aed14 --- /dev/null +++ b/src/artifacts.ts @@ -0,0 +1,5367 @@ +import * as path from 'node:path' +import { Duplex, Readable } from 'node:stream' +import { FileHandle, Fs, FsEntity, FsEntityStats, JsonFs, SyncFs, watchForFile } from './system' +import type { ExternalValue, PackageInfo } from './runtime/modules/serdes' +import { getLogger, runTask } from './logging' +import { Mutable, acquireFsLock, createRwMutex, createTrie, deepClone, getHash, isNonNullable, keyedMemoize, memoize, sortRecord, throwIfNotFileNotFoundError, tryReadJson } from './utils' +import type { TerraformPackageManifest, TfJson } from './runtime/modules/terraform' +import { BuildTarget, Deployment, Program, getBuildDir, getProgramIdFromDeployment, getRootDir, getRootDirectory, getTargetDeploymentIdOrThrow, getWorkingDir, getWorkingDirectory, resolveProgramBuildTarget } from './workspaces' +import { NpmPackageInfo } from './pm/packages' +import { TargetsFile, readPointersFile } from './compiler/host' +import { TarballFile } from './utils/tar' +import { getBuildTarget, getBuildTargetOrThrow, getExecutionId, getFs, throwIfCancelled } from './execution' +import { TypeInfo, TypesFileData, getTypesFile } from './compiler/resourceGraph' +import { DataPointer, Pointers, applyPointers, createPointer, extractPointers, getEmptyObjectHash, getNullHash, isDataPointer, isNullHash, isNullMetadataPointer, pointerPrefix, toAbsolute, toDataPointer } from './build-fs/pointers' +import { TfState } from './deploy/state' +import { printLine } from './cli/ui' +import { createBlock, getBlockInfo, openBlock } from './build-fs/block' +import { FlatImportMap, SourceInfo } from './runtime/importMaps' +import { RemoteArtifactRepository, createRemoteArtifactRepo } from './build-fs/remote' + +export interface GlobalMetadata { + /** @deprecated */ + readonly multiPart?: { + readonly parts: string[] + readonly chunkSize: number + } +} + +export interface LocalMetadata { + readonly name?: string + readonly source?: string + readonly moduleId?: string + readonly sourcemaps?: Record + readonly publishName?: string + readonly dependencies?: string[] + readonly pointers?: Pointers + + readonly packageDependencies?: any + + readonly sourceDelta?: { line: number; column: number } + // TODO: use a generic `data` field for everything except dependencies/pointers +} + +export interface ArtifactMetadata extends GlobalMetadata, Omit { + readonly dependencies?: Record +} + +export interface Head { + readonly id: string + readonly timestamp: string + readonly storeHash: string + readonly programHash?: string + readonly isTest?: boolean + readonly isRollback?: boolean + readonly previousCommit?: string +} + +export interface SerializedTemplate { + readonly '//'?: string + readonly provider: string + readonly resource: Record + readonly data: Record + readonly terraform: string + readonly locals: Record + readonly moved?: { from: string; to: string }[] +} + +export interface SerializedState { + readonly serial: number + readonly version: number + readonly lineage: string + readonly resources: Record +} + + +export interface DeployedModule { + readonly kind: 'deployed' + readonly table: Record + readonly captured: any + readonly rendered?: string +} + +export interface CompiledChunk { + readonly kind: 'compiled-chunk' + readonly runtime: string + readonly infra: string +} + +export type Artifact = + | DeployedModule + | CompiledChunk + +const isPointer = (fileName: string) => fileName.startsWith(pointerPrefix) +const hasMetadata = (fileName: string) => fileName.length === pointerPrefix.length + 129 + +function getArtifactName(target: string) { + if (isDataPointer(target)) { + return target.hash + } + + if (!isPointer(target)) { + throw new Error(`Not an artifact: ${target}`) + } + + return target.slice(pointerPrefix.length) +} + +const chunkSize = 4 * 1024 * 1024 // 4 MB + +interface BaseArtifactStore { + getMetadata(hash: string, source: string): ArtifactMetadata + readArtifact(hash: string): Promise + readArtifactSync(hash: string): Uint8Array + listArtifacts(): Promise>> + listArtifactsSync(): Record> + + // listDependencies(): Promise> + resolveMetadata(metadata: LocalMetadata): ArtifactMetadata +} + +export interface ClosedArtifactStore = Record> extends BaseArtifactStore { + readonly state: 'closed' + readonly hash: string +} + +export interface OpenedArtifactStore = Record> extends BaseArtifactStore { + readonly id: string + readonly state: 'opened' + setMetadata(hash: string, metadata: ArtifactMetadata): void + writeArtifact(data: Uint8Array, metadata?: ArtifactMetadata): Promise + writeArtifactSync(data: Uint8Array, metadata?: ArtifactMetadata): string + close(): string +} + +export type ArtifactStore = Record> = OpenedArtifactStore | ClosedArtifactStore + +interface DenormalizedStoreManifest { + readonly type?: 'denormalized' + readonly artifacts: Record> +} + +interface FlatStoreManifest { + readonly type: 'flat' + readonly artifacts: Record +} + +type ArtifactStoreManifest = DenormalizedStoreManifest | FlatStoreManifest + +interface ArtifactStoreState2 { + readonly hash?: string + readonly status: 'opened' | 'closed' + readonly mode: 'flat' + readonly workingMetadata: FlatStoreManifest['artifacts'] +} + +interface ArtifactStoreState3 { + readonly hash?: string + readonly status: 'opened' | 'closed' + readonly mode: 'denormalized' + readonly workingMetadata: DenormalizedStoreManifest['artifacts'] +} + + +type ArtifactStoreState = ArtifactStoreState2 | ArtifactStoreState3 + +function initArtifactStoreState(): ArtifactStoreState { + return { + status: 'opened', + mode: 'flat', + workingMetadata: {}, + } +} + +function cow>(val: T): T { + let copied: Partial | undefined + function getClone() { + return copied ??= deepClone(val) + } + + if (val instanceof Map) { + return new Proxy(val, { + get: (_, prop, recv) => { + if (copied) { + return Reflect.get(copied, prop, recv) + } + + if (prop === 'set' || prop === 'delete' || prop === 'clear') { + return (...args: any[]) => (Reflect.get(getClone(), prop, recv) as any)(...args) + } + + return Reflect.get(val, prop, recv) + }, + }) + } + + if (val instanceof Set || val instanceof Array) { + throw new Error(`Not implemented`) + } + + return new Proxy(val, { + get: (_, prop, recv) => { + if (copied === undefined) { + copied = {} as T + } + if (prop in copied) { + return copied[prop as any] + } + + const v = val[prop as any] + + return (copied as any)[prop] = typeof v !== 'object' || !v ? v : cow(v) + }, + set: (_, prop, newValue, recv) => { + if (copied === undefined) { + copied = {} as T + } + (copied as any)[prop] = newValue + + return true + }, + }) as T +} + +// Artifact store 3.0 (call this a "build unit"?) +// * Inputs -> named committed stores +// * Output -> pruned & committed store +// * Attributes (metadata) + +interface ResolveArtifactOpts { + name?: string + extname?: string + filePath?: string + noWrite?: boolean +} + +interface RootArtifactStore { + commitStore(state: ArtifactStoreState): string + createStore>(): OpenedArtifactStore + getStore>(hash: string): Promise> + getStoreSync>(hash: string): ClosedArtifactStore + + getBuildFs(hash: string): Promise + + readData(hash: string): Promise + readDataSync(hash: string): Uint8Array + + writeData(hash: string, data: Uint8Array): Promise + writeDataSync(hash: string, data: Uint8Array): void + + getMetadata(hash: string): Record | undefined + getMetadata(hash: string, source: string): ArtifactMetadata + getMetadata2(pointer: string): Promise + + resolveArtifact(hash: string, opt?: ResolveArtifactOpts): Promise | string + + deleteData(hash: string): Promise + deleteDataSync(hash: string): void +} + +// function getHash(data: Uint8Array) { +// const h = createHash('sha256') +// h.write(data) +// h.end() + +// return new Promise((resolve, reject) => { +// h.on('data', (d: Buffer) => { +// resolve(d.toString('hex')) +// }) +// h.on('error', reject) +// }) +// } + +// function sortArtifacts(artifacts: Record): Record { +// // First sort by content length +// // Break ties with hash +// // Then do a topo sort + +// const presort = Object.entries(artifacts).sort((a, b) => { +// const c = a[1].contentLength! - b[1].contentLength! +// if (c !== 0) { +// return c +// } +// return a[0].localeCompare(b[0]) +// }) + +// // Empty string is the root node +// const edges: [string, string][] = [] +// for (const [k, v] of presort) { +// edges.push(['', k]) +// if (v.dependencies) { +// for (const d of v.dependencies) { +// edges.push([k, d]) +// } +// } +// } + +// const sorted = topoSort(edges).slice(1).reverse() +// console.log(sorted) + +// return Object.fromEntries(sorted.map(h => [h, artifacts[h]] as const)) +// } + +function createReverseIndex() { + const reverseIndex: Record> = {} + const indexQueue: Record[]> = {} + const sourceAliases: Record = {} + + function search(hash: string, source?: string) { + if (source) { + const allSources = [source, ...(sourceAliases[source] ?? [])] + for (const s of allSources) { + const v = indexQueue[s] + const m = v ? searchQueue(s, v, hash) : reverseIndex[hash]?.[s] + if (m) { + return m + } + } + + return + } + + for (const [k, v] of Object.entries(indexQueue)) { + const m = searchQueue(k, v, hash) + if (m) { + return m + } + } + } + + function searchQueue(k: string, v: Record[], hash: string) { + delete indexQueue[k] + + while (v.length > 0) { + const n = v.pop()! + const m = searchAndIndex(k, n, hash) + if (m) { + if (v.length > 0) { + indexQueue[k] = v + } + + return m + } + } + } + + function searchAndIndex(source: string, artifacts: Record, targetHash: string) { + let found: ArtifactMetadata | undefined + for (const [k, v] of Object.entries(artifacts)) { + const o = reverseIndex[k] ??= {} + if (!o[source]) { + o[source] = v + } else { + // TODO: assert that `v` is equivalent to `o[source]` + } + + if (k === targetHash) { + found = o[source] + } + } + + return found + } + + function index(source: string, artifacts: Record>) { + if (hasIndex(source)) { + throw new Error(`Store "${source}" was already indexed`) + } + + const aliases = sourceAliases[source] = [] as string[] + for (const [k, v] of Object.entries(artifacts)) { + if (k === '') { + throw new Error(`Artifact index is missing source hash: ${source}`) + } + + const arr = indexQueue[k] ??= [] + arr.push(v) + + if (k !== source) { + aliases.push(k) + } + } + } + + function get(hash: string): Record | undefined + function get(hash: string, source: string): ArtifactMetadata | undefined + function get(hash: string, source?: string) { + if (!source) { + if (!reverseIndex[hash]) { + return search(hash, source) + } + + return reverseIndex[hash] + } + + return reverseIndex[hash]?.[source] ?? search(hash, source) + } + + function hasIndex(source: string) { + return !!sourceAliases[source] + } + + return { get, index, hasIndex } +} + +class MetadataConflictError extends Error { + public constructor(public readonly dataHash: string) { + super(`Store already has metadata for data: ${dataHash}`) + } +} + +let storeIdCounter = 0 +// Stores are strictly additive +// They should not be re-used for the same operation. But they should be re-used for subsequent operations. +function createArtifactStore(root: RootArtifactStore, state = initArtifactStoreState()) { + const id = `${storeIdCounter++}` + + function close() { + ;(state as Mutable).status = 'closed' + + return root.commitStore(state) + } + + function getMetadata(hash: string, source: string): ArtifactMetadata { + if (state.mode === 'flat') { + if (source === (state.hash ?? '')) { + const m = state.workingMetadata[hash] + if (m) { + return m + } + } + } else { + const m = state.workingMetadata[source]?.[hash] + if (m) { + return m + } + } + + throw new Error(`No metadata found for ${hash} from ${source}`) + } + + function setMetadata(hash: string, metadata: ArtifactMetadata): void { + if (state.status === 'closed') { + throw new Error(`Cannot write after closed`) + } + + if (hasMetadata(hash, '')) { + throw new MetadataConflictError(hash) + } + + if (state.mode === 'flat') { + state.workingMetadata[hash] = metadata + } else { + const m = state.workingMetadata[''] ??= {} + m[hash] = metadata + + if (metadata.dependencies) { + ensureDeps(state.workingMetadata, metadata.dependencies, '') + } + } + } + + function ensureDeps(workingMetadata: DenormalizedStoreManifest['artifacts'], deps: Record, source?: string) { + for (const [k, v] of Object.entries(deps)) { + const s = k || source + if (!s) continue + + for (const d of v) { + const wm = workingMetadata[s] ??= {} + if (wm[d]) continue + + const m = root.getMetadata(d, s) + + wm[d] = m + if (m.dependencies) { + ensureDeps(workingMetadata, m.dependencies, s) + } + } + } + } + + async function splitArtifact(data: Uint8Array) { + const parts: [string, Buffer][] = [] + for await (const part of createChunkStream(Buffer.from(data))) { + parts.push([part.hash, part.chunk]) + } + + return { parts, chunkSize } + } + + async function writeMultipart(hash: string, data: Uint8Array, metadata: ArtifactMetadata) { + const { parts, chunkSize } = await splitArtifact(data) + + setMetadata(hash, { + ...metadata, + multiPart: { parts: parts.map(x => x[0]), chunkSize }, + }) + + await root.writeData(hash, data) + await Promise.all(parts.map(([hash, data]) => write(hash, data))) + } + + function write(hash: string, data: Uint8Array, metadata: ArtifactMetadata = {}) { + setMetadata(hash, metadata) + + return root.writeData(hash, data) + } + + async function writeArtifact(data: Uint8Array, metadata?: ArtifactMetadata): Promise { + const hash = getHash(data) + await write(hash, data, metadata) + + return hash + } + + function writeArtifactSync(data: Uint8Array, metadata?: ArtifactMetadata): string { + const hash = getHash(data) + write(hash, data, metadata) + + return hash + } + + async function readArtifact(name: string): Promise { + return root.readData(name) + } + + function readArtifactSync(name: string): Uint8Array { + return root.readDataSync(name) + } + + function listArtifactsSync() { + if (state.mode === 'flat') { + return { [state.hash ?? '']: state.workingMetadata } + } + + return state.workingMetadata + } + + async function listArtifacts() { + return listArtifactsSync() + } + + function hasMetadata(hash: string, source: string) { + if (state.mode === 'flat') { + if (source !== (state.hash ?? '')) { + return false + } + + return !!state.workingMetadata[hash] + } + + return !!state.workingMetadata?.[source]?.[hash] + } + + function resolveMetadata(metadata: LocalMetadata) { + const dependencies: Record = {} + if (metadata.dependencies) { + for (const d of metadata.dependencies) { + if (isDataPointer(d)) { + if (d.isResolved() || !d.isContainedBy(id)) { + const { hash, storeHash } = d.resolve() + const arr = dependencies[storeHash] ??= [] + arr.push(hash) + } else { + const arr = dependencies[''] ??= [] + arr.push(d.hash) + } + } else if (d.length === 129) { // ${sha256}:${sha256} === 129 characters + if (d[64] !== ':') { + throw new Error(`Malformed dependency string: ${d}`) + } + + const arr = dependencies[d.slice(0, 64)] ??= [] + arr.push(d.slice(65)) + } else { + throw new Error(`Unresolved dependency: ${d}`) + } + } + } + + const sourcemaps: Record = {} + if (metadata.sourcemaps) { + for (const [k, v] of Object.entries(metadata.sourcemaps)) { + // This effectively excludes sourcemaps from the dependency graph + sourcemaps[k] = isDataPointer(v) ? toAbsolute(v) : v + } + } + + const result = { + ...metadata, + sourcemaps: Object.keys(sourcemaps).length > 0 ? sourcemaps : undefined, + dependencies: Object.keys(dependencies).length > 0 ? dependencies : undefined, + } + + const pruned = Object.fromEntries(Object.entries(result).filter(([_, v]) => v !== undefined)) + + return pruned + } + + return { + id, + hash: state.hash, + state: state.status, + close, + listArtifacts, + listArtifactsSync, + writeArtifact, + writeArtifactSync, + readArtifact, + readArtifactSync, + + getMetadata, + setMetadata, + resolveMetadata, + } as ArtifactStore +} + +export function extractPointersFromResponse(resp: any) { + const [state, pointers] = extractPointers(resp) + + return { state, pointers } +} + +export interface ReadonlyBuildFs { + readonly hash: string + readonly index: BuildFsIndex +} + +interface OpenOptions { + readonly readOnly?: boolean + readonly writeToDisk?: boolean + readonly clearPrevious?: boolean +} + +export interface BuildFsFragment { + open(key: string, opt?: OpenOptions): BuildFsFragment + clear(key?: string): void + mount(path: string, fs: ReadonlyBuildFs): void + + writeFile(fileName: string, data: string | Uint8Array, metadata?: LocalMetadata): Promise + writeFileSync(fileName: string, data: string | Uint8Array, metadata?: LocalMetadata): void + writeData(data: Uint8Array, metadata?: LocalMetadata): Promise + writeDataSync(data: Uint8Array, metadata?: LocalMetadata): DataPointer + + readFile(fileName: string): Promise + readFile(fileName: string, encoding: BufferEncoding): Promise + + readFileSync(fileName: string): Uint8Array + readFileSync(fileName: string, encoding: BufferEncoding): string + + readData(hash: string): Promise + readDataSync(hash: string): Uint8Array + + fileExistsSync(fileName: string): boolean + dataExistsSync(hash: string): boolean + + getMetadata(fileName: string, dataHash?: string): Promise + getMetadataSync(fileName: string, dataHash?: string): ArtifactMetadata + + listFiles(): Promise<{ name: string; hash: string }[]> + + // BACKWARDS COMPAT + writeArtifact(data: Uint8Array, metadata?: LocalMetadata): Promise + writeArtifactSync(data: Uint8Array, metadata?: LocalMetadata): string + readArtifact(pointer: string): Promise + readArtifactSync(pointer: string): Uint8Array + resolveArtifact(pointer: string, opt?: ResolveArtifactOpts): Promise + + writeData2(data: any, metadata?: LocalMetadata): Promise + readData2(pointer: string): Promise +} + +interface BuildFsFile { + readonly hash: string + readonly store: string + readonly storeHash?: string + // readonly mount?: string +} + +interface BuildFsStore { + readonly hash: string +} + +export interface BuildFsIndex { + readonly files: Record + readonly stores: Record + readonly dependencies?: Record // Extra FS hashes that should be accounted for during GC +} + +interface DumpFsOptions { + readonly link?: boolean + readonly clean?: boolean + readonly prettyPrint?: boolean + readonly writeIndex?: boolean +} + +async function dump(repo: DataRepository, index: BuildFsIndex & { id?: string }, dest: string, opt?: DumpFsOptions) { + const fs = getFs() + + async function doClean() { + await fs.deleteFile(dest).catch(throwIfNotFileNotFoundError) + } + + if (opt?.clean) { + await doClean() + } + + const indexPath = path.resolve(dest, '__index__.json') + const oldIndex = !opt?.clean ? await tryReadJson(fs, indexPath) : undefined + + if (opt?.writeIndex && !opt?.clean && !oldIndex) { + if (await fs.fileExists(dest)) { + getLogger().debug(`Removing corrupted linked package`) + await fs.deleteFile(dest) + } + } + + let shouldIgnoreOld = false + if (oldIndex && oldIndex.id !== index.id) { + getLogger().debug(`Removing old linked package`) + await doClean() + shouldIgnoreOld = true + } + + async function worker(k: string, v: BuildFsFile) { + const oldHash = oldIndex?.files[k]?.hash + if (!shouldIgnoreOld && oldHash === v.hash) { + return + } + + const destPath = path.resolve(dest, k) + if (oldHash && oldHash !== v.hash) { + await fs.deleteFile(destPath).catch(throwIfNotFileNotFoundError) + } + + const fp = await repo.resolveArtifact(v.hash, { noWrite: true }) + if (k.endsWith('.json') && opt?.prettyPrint) { + const data = JSON.parse(decode(await repo.readData(v.hash), 'utf-8')) + await fs.writeFile(destPath, JSON.stringify(data, undefined, 4)) + } else if (opt?.link) { + await fs.link(fp, destPath).catch(async e => { + if ((e as any).code !== 'EEXIST') { + throw e + } + await fs.deleteFile(destPath) + await fs.link(fp, destPath) + }) + } else { + await fs.writeFile(destPath, await repo.readData(v.hash)) + } + } + + await Promise.all(Object.entries(index.files).map(([k, v]) => worker(k, v))) + + if (oldIndex && !shouldIgnoreOld) { + for (const [k, v] of Object.entries(oldIndex.files)) { + if (!(k in index.files)) { + await fs.deleteFile(path.resolve(dest, k)).catch(throwIfNotFileNotFoundError) + } + } + } + + if (opt?.writeIndex) { + await fs.writeFile(indexPath, JSON.stringify(index, undefined, 4)) + } +} + +export interface WriteFileOptions { + readonly encoding?: BufferEncoding + readonly flag?: 'w' | 'wx' // | 'w+' | 'wx+' + + /** Using `#mem` writes a temporary in-memory file */ + readonly fsKey?: '#mem' | string + readonly metadata?: LocalMetadata + readonly checkChanged?: boolean +} + +// `fsKey` format: +// [ ${fsId}: ]${storeKey} + +function parseFsKey(key: string) { + const splitIndex = key.indexOf(':') + if (splitIndex === -1) { + return { storeKey: key } + } + + const fsId = key.slice(0, splitIndex) + const storeKey = key.slice(splitIndex + 1) + + return { fsId, storeKey } +} + +// [#${fsKey}]${fileName} +function hasBuildFsPrefix(fileName: string) { + return fileName.startsWith('[#') +} + +function parseBuildFsPrefix(fileName: string) { + const match = fileName.match(/^\[#([^\[#\]]+)\](.*)/) + if (!match) { + throw new Error(`Failed to parse prefix from name: ${fileName}`) + } + + const keyComponents = parseFsKey(match[1]) + const actualFileName = match[2] + + return { + fileName: actualFileName, + fsId: keyComponents.fsId, + storeKey: keyComponents.storeKey, + } +} + +export interface ReadFileOptions { + readonly encoding?: BufferEncoding + readonly fsKey?: '#mem' | string +} + +function decode(data: Uint8Array, encoding: BufferEncoding) { + const buf = Buffer.isBuffer(data) ? data : Buffer.from(data) + + return buf.toString(encoding) +} + +interface OpenedBuildFs { + key: string + store: OpenedArtifactStore + files: Record + pointers: Record + options?: OpenOptions + didChange?: boolean +} + +function fixWindowsPath(p: string) { + return p.replaceAll('\\', '/') +} + +/** @internal */ +export function createBuildFsFragment(repo: DataRepository, index: BuildFsIndex, copier = createCopier(repo)) { + const opened: Record = {} + const mounts: Record = {} + const dependencies: Record = { ...index.dependencies } + const storeHashes: Record = {} + let didIndexChange = false + + function searchMounts(fileName: string) { + for (const [k2, v2] of Object.entries(mounts)) { + if (fileName.startsWith(k2)) { + const rel = path.posix.relative(k2, fileName) + const f = v2.index.files[rel] + if (f) { + return { store: k2 + '/' + f.store, fileHash: f.hash, storeHash: f.storeHash ?? v2.index.stores[f.store].hash } + } + } + } + } + + function findOpenedFile(fileName: string) { + // TODO: use flat list to track opened files + for (const [k, v] of Object.entries(opened)) { + const hash = v.files[fileName] + if (hash) { + return { store: k, fileHash: hash } + } + } + } + + function findFile(fileName: string, useMounts = true) { + const f = index.files[fileName] + if (f) { + return { fileHash: f.hash, store: f.store, storeHash: f.storeHash ?? index.stores[f.store].hash } + } + + return useMounts ? searchMounts(fileName) : undefined + } + + function getSubstores(key: string) { + const prefix = key === '/' ? key : `${key}/` + + return Object.keys(index.stores).filter(k => k.startsWith(prefix)) + } + + function getFilesFromStore(key: string) { + const res: Record = {} + for (const [k, v] of Object.entries(index.files)) { + if (v.store === key) { + res[k] = v.hash + } + } + return res + } + + function renameStore(from: string, to: string) { + if (from === to) { + return + } + + assertOpen() + + const moves: [from: string, to: string][] = [] + function addMove(from: string, to: string) { + if (opened[from]) { + throw new Error(`Cannot rename an opened store: ${from}`) + } + if (opened[to] || index.stores[to]) { + throw new Error(`Cannot rename to an existing store: ${to}`) + } + if (!index.stores[from]) { + throw new Error(`No such store exists: ${from}`) + } + moves.push([from, to]) + } + + addMove(from, to) + for (const fromKey of getSubstores(from)) { + const toKey = `${to}${fromKey.slice(from.length)}` + addMove(fromKey, toKey) + } + + for (const [fromKey, toKey] of moves) { + index.stores[toKey] = index.stores[fromKey] + for (const [k, v] of Object.entries(getFilesFromStore(fromKey))) { + index.files[k] = { + hash: v, + store: toKey, + storeHash: index.files[k].storeHash, + } + } + } + + didIndexChange = true + } + + function assertOpen() { + if (closed) { + throw new Error(`Build fs already closed`) + } + } + + let closed = false + function flush() { + assertOpen() + closed = true + return _flush() + } + + function updateIndex(): BuildFsIndex | undefined { + const pending = Object.keys(opened) + for (const key of pending) { + _close(key) + } + + // This could be more granular by checking if each component changed + if (!didIndexChange) { + return + } + + return { + files: sortRecord(index.files), + stores: sortRecord(index.stores), + dependencies: sortRecord(dependencies), + } + } + + async function _flush() { + const newIndex = updateIndex() + if (newIndex === undefined) { + return + } + + return await writeJsonRaw(repo, newIndex) + } + + function _close(key: string) { + const data = opened[key] + delete opened[key] + stores.delete(key) + + if (!data.didChange && !data.options?.clearPrevious) { + return storeHashes[data.store.id] = index.stores[key]?.hash ?? getNullHash() + } + + return commit(key, data) + } + + function commit(key: string, data = getOpenedOrThrow(key)) { + const hash = data.store.close() + const oldHash = index.stores[key]?.hash + if (oldHash === hash) { + return storeHashes[data.store.id] = hash + } + + index.stores[key] = { hash } + didIndexChange = true + + for (const [k, v] of Object.entries(data.files)) { + index.files[k] = { + hash: v, + store: key, + } + } + + for (const [k, v] of Object.entries(index.files)) { + if (v.store === key && !v.storeHash && !data.options?.clearPrevious && !(k in data.files)) { + index.files[k] = { + ...index.files[k], + storeHash: oldHash, + } + } + } + + return storeHashes[data.store.id] = hash + } + + function getStoreByKey(key: string) { + const s = index.stores[key] + if (!s) { + throw new Error(`No store found: ${key}`) + } + return s + } + + async function getStore(key: string) { + return repo.getStore(getStoreByKey(key).hash) + } + + function getStoreSync(key: string) { + return repo.getStoreSync(getStoreByKey(key).hash) + } + + function deleteStore(key: string) { + const shouldDelete = new Set() + + function checkForRemoval(k: string) { + if (!!index.stores[k] || !!opened[k]) { + shouldDelete.add(k) + } + } + + checkForRemoval(key) + for (const k of getSubstores(key)) { + checkForRemoval(k) + } + + if (shouldDelete.size === 0) { + return + } + + assertOpen() + + for (const [k, v] of Object.entries(index.files)) { + if (shouldDelete.has(v.store)) { + delete index.files[k] + } + } + + for (const k of shouldDelete) { + delete index.stores[k] + delete opened[k] + } + + didIndexChange = true + } + + function listStores() { + return index.stores + } + + function listFiles(key?: string) { + const resolvedKey = key && !key.startsWith('/') ? `/${key}` : key + const allFiles: Record = {} + for (const [k, v] of Object.entries(index.files)) { + if (resolvedKey && !v.store.startsWith(resolvedKey)) continue + + allFiles[k] = { + hash: v.hash, + store: { name: v.store, hash: v.storeHash ?? index.stores[v.store].hash }, + } + } + + return allFiles + } + + async function resolveArtifact(pointer: string, opt?: ResolveArtifactOpts) { + const hash = getArtifactName(pointer) + + return repo.resolveArtifact(hash, opt) + } + + function mount(path: string, fs: ReadonlyBuildFs) { + if (process.platform === 'win32') { + path = fixWindowsPath(path) + } + + mounts[path] = fs + if (index.dependencies?.[path] !== fs.hash) { + assertOpen() + dependencies[path] = fs.hash + didIndexChange = true + } + } + + function unmount(path: string) { + if (process.platform === 'win32') { + path = fixWindowsPath(path) + } + + if (mounts[path]) { + assertOpen() + delete mounts[path] + if (dependencies[path]) { + delete dependencies[path] + didIndexChange = true + } + } + } + + function readDataSync(hash: string, encoding: BufferEncoding) { + return encoding ? decode(repo.readDataSync(hash), encoding) : repo.readDataSync(hash) + } + + function _resolveMetadata(target: OpenedBuildFs, metadata?: LocalMetadata): ArtifactMetadata | undefined + function _resolveMetadata(target: OpenedBuildFs, metadata: LocalMetadata): ArtifactMetadata + function _resolveMetadata(target: OpenedBuildFs, metadata?: LocalMetadata) { + return metadata ? target.store.resolveMetadata(metadata) : undefined + } + + function _write(target: OpenedBuildFs, data: Uint8Array, metadata: LocalMetadata | undefined, isAsync?: false): DataPointer + function _write(target: OpenedBuildFs, data: Uint8Array, metadata: LocalMetadata | undefined, isAsync: true): Promise + function _write(target: OpenedBuildFs, data: Uint8Array, metadata: LocalMetadata | undefined, isAsync?: boolean) { + target.didChange = true // TODO: would it be faster to check the # of files on close? + + const resolved = _resolveMetadata(target, metadata) + if (resolved === undefined || Object.keys(resolved).length === 0) { + // No metadata means we don't need to use a store + const dataHash = getHash(data) + const np = createPointer(dataHash, getNullHash()) + + if (isAsync) { + return repo.writeData(dataHash, data).then(() => np) + } + + repo.writeDataSync(dataHash, data) + return np + } + + function onEnd(hash?: string, err?: unknown) { + if (hash !== undefined) { + return _getPointer(target, hash) + } + + if (!(err instanceof MetadataConflictError)) { + throw err + } + + _close(target.key) + const newTarget = initStore(target.key) + newTarget.store.setMetadata(err.dataHash, resolved ?? {}) + + return _getPointer(newTarget, err.dataHash) + } + + try { + if (isAsync) { + return target.store.writeArtifact(data, resolved) + .then(h => onEnd(h)) + .catch(e => onEnd(undefined, e)) + } + + return onEnd(target.store.writeArtifactSync(data, resolved)) + } catch (e) { + return onEnd(undefined, e) + } + } + + async function _writeFile(target: OpenedBuildFs, fileName: string, data: Uint8Array, metadata?: LocalMetadata) { + const p = await _write(target, data, metadata, true) + + if (isNullMetadataPointer(p)) { + index.files[fileName] = { + hash: p.hash, + store: target.key, + storeHash: getNullHash(), + } + didIndexChange = true + } else { + target.files[fileName] = p.hash + } + + return p + } + + function _writeFileSync(target: OpenedBuildFs, fileName: string, data: Uint8Array, metadata?: LocalMetadata) { + const p = _write(target, data, metadata, false) + + if (isNullMetadataPointer(p)) { + index.files[fileName] = { + hash: p.hash, + store: target.key, + storeHash: getNullHash(), + } + didIndexChange = true + } else { + target.files[fileName] = p.hash + } + + return p + } + + function _writeData(target: OpenedBuildFs, data: Uint8Array, metadata?: LocalMetadata) { + return _write(target, data, metadata, true) + } + + function _writeDataSync(target: OpenedBuildFs, data: Uint8Array, metadata?: LocalMetadata) { + return _write(target, data, metadata, false) + } + + function _writeData2(target: OpenedBuildFs, data: any, metadata?: LocalMetadata) { + if (metadata && 'pointers' in metadata) { + return _write(target, Buffer.from(JSON.stringify(data), 'utf-8'), metadata, true) + } + + const [res, pointers] = extractPointers(data) + const m = { ...metadata, pointers } + + return _write(target, Buffer.from(JSON.stringify(res), 'utf-8'), m, true) + } + + async function _readData2(pointer: string, from?: OpenedBuildFs) { + if (isDataPointer(pointer)) { + const metadata = await repo.getMetadata2(pointer) + + const data = await repo.readData(pointer.hash) + const parsed = JSON.parse(decode(data, 'utf-8')) + if (!metadata.pointers) { + return parsed + } + + return applyPointers(parsed, metadata.pointers) + } + + const hash = getArtifactName(pointer) + const data = await repo.readData(hash) + + return JSON.parse(decode(data, 'utf-8')) + } + + function _createPointer(target: OpenedBuildFs, dataHash: string) { + const id = target.store.id + + return createPointer(dataHash, { + id, + close: () => storeHashes[id] ??= _close(target.key) + }) + } + + function _getPointer(target: OpenedBuildFs, dataHash: string) { + return target.pointers[dataHash] ??= _createPointer(target, dataHash) + } + + async function _readFile(fileName: string, encoding?: BufferEncoding, from?: OpenedBuildFs) { + const newFile = findOpenedFile(fileName) + if (newFile) { + // if (newFile.store === from?.key) { + // const d = await repo.readData(newFile.fileHash) + // return encoding ? decode(d, encoding) : d + // } + + // getLogger().debug(`Committing store "${newFile.store}" early from reading "${fileName}"`) + + // const hash = commit(newFile.store) + // from?.contexts.add(hash) + + const d = await repo.readData(newFile.fileHash) + return encoding ? decode(d, encoding) : d + } + + const oldFile = findFile(fileName) + if (!oldFile) { + throw Object.assign(new Error(`No file found: ${fileName}`), { code: 'ENOENT' }) + } + + const d = await repo.readData(oldFile.fileHash) + return encoding ? decode(d, encoding) : d + } + + function _readFileSync(fileName: string, encoding?: BufferEncoding, from?: OpenedBuildFs) { + const newFile = findOpenedFile(fileName) + if (newFile) { + const d = repo.readDataSync(newFile.fileHash) + return encoding ? decode(d, encoding) : d + } + + const fileHash = from?.files[fileName] + if (fileHash) { + return repo.readDataSync(fileHash) + } + + const oldFile = findFile(fileName) + if (!oldFile) { + throw Object.assign(new Error(`No file found: ${fileName}`), { code: 'ENOENT' }) + } + + const d = repo.readDataSync(oldFile.fileHash) + return encoding ? decode(d, encoding) : d + } + + async function _copyFile(target: OpenedBuildFs, existingPath: string, newPath: string) { + const f = findFile(existingPath) + if (!f) { + throw Object.assign(new Error(`No file found: ${existingPath}`), { code: 'ENOENT' }) + } + + const oldFile = findFile(newPath) + if (oldFile && oldFile.store !== target.key) { + throw new Error(`File "${newPath}" already exists [store: ${oldFile.store} -> ${oldFile.storeHash}]`) + } + + const p = await copier.copyData(f.fileHash, f.storeHash, (d, m) => { + return _write(getOrCreateStore(target.key), d, m, true) + }) + + if (isNullMetadataPointer(p)) { + index.files[newPath] = { + hash: p.hash, + store: target.key, + storeHash: getNullHash(), + } + didIndexChange = true + } else { + getOpenedOrThrow(target.key).files[newPath] = p.hash + } + } + + function fileExistsSync(fileName: string) { + return !!findFile(fileName) || !!findOpenedFile(fileName) + } + + function dataExistsSync(hash: string) { + return repo.hasDataSync(hash) + } + + function __getMetadataWorker(fileName: string, dataHash?: string, from?: string) { + if (isPointer(fileName)) { + const hash = getArtifactName(fileName) + + const storeHash = from ? getStoreByKey(from).hash : undefined + const m = storeHash ? repo.getMetadata(hash, storeHash) : repo.getMetadata(hash) + if (!m) { + throw new Error(`No metadata found for file: ${fileName}${storeHash ? ` [store: ${storeHash}]` : ''}`) + } + + const entry = (!storeHash ? Object.entries(m).shift()! : [storeHash, m]) as [string, ArtifactMetadata] + + return entry[1] + } + + const f = findFile(fileName) + if (!f) { + throw Object.assign(new Error(`No file found: ${fileName}`), { code: 'ENOENT' }) + } + + const objHash = dataHash ?? f.fileHash + const m = repo.getMetadata(objHash, f.storeHash) + if (!m) { + throw new Error(`No metadata found for file: ${fileName} [hash: ${objHash}; store: ${f.storeHash}]`) + } + + return m + } + + async function _getMetadata(fileName: string, dataHash?: string, from?: string) { + return _getMetadataSync(fileName, dataHash, from) + } + + function _getMetadataSync(fileName: string, dataHash?: string, from?: string) { + const newFile = findOpenedFile(fileName) + if (!newFile) { + return __getMetadataWorker(fileName, dataHash, from) + } + + const target = opened[newFile.store] + const s = target.store + const objHash = dataHash ?? newFile.fileHash + const m = s.getMetadata(objHash, '') + + return m + } + + function getPointer(fileName: string) { + const newFile = findOpenedFile(fileName) + if (newFile) { + const target = opened[newFile.store] + + return target.pointers[newFile.fileHash] ?? createPointer(newFile.fileHash, getNullHash()) + } + + const f = findFile(fileName) + if (!f) { + throw Object.assign(new Error(`No file found: ${fileName}`), { code: 'ENOENT' }) + } + + return createPointer(f.fileHash, f.storeHash) + } + + function initStore(key: string, options?: OpenOptions) { + if (opened[key]) { + throw new Error(`Store "${key}" already initialized`) + } + + return opened[key] = { + key, + files: {}, + pointers: {}, + store: repo.createStore(), + options, + } + } + + function getOpenedOrThrow(key: string) { + const data = opened[key] + if (!data) { + throw new Error(`No store opened with key: ${key}`) + } + return data + } + + function getOrCreateStore(key: string, opt?: OpenOptions) { + return opened[key] ?? initStore(key, opt) + } + + function _clear(key: string) { + deleteStore(key) + } + + function joinSubkey(key: string, subkey: string) { + if (subkey.endsWith('/') && subkey !== '/') { + throw new Error(`Key cannot end with a slash: ${subkey}`) + } + + if (!subkey.startsWith('/')) { + return `${key === '/' ? '' : key}/${subkey}` + } + + if (!subkey.startsWith(`${key === '/' ? key : `${key}/`}`)) { + throw new Error(`Cannot open subkey "${subkey}" under "${key}"`) + } + + return subkey + } + + function openStore(key: string, opt?: OpenOptions) { + const isReadOnly = opt?.readOnly + const getTarget = isReadOnly ? () => getOpenedOrThrow(key) : () => getOrCreateStore(key, opt) + const maybeGetTarget = isReadOnly ? () => opened[key] as OpenedBuildFs | undefined : () => getOrCreateStore(key, opt) + + async function writeFile(fileName: string, data: Uint8Array | string, metadata?: LocalMetadata) { + const buffer = typeof data === 'string' ? Buffer.from(data) : data + return _writeFile(getTarget(), fileName, buffer, metadata) + } + + function writeFileSync(fileName: string, data: Uint8Array | string, metadata?: LocalMetadata) { + const buffer = typeof data === 'string' ? Buffer.from(data) : data + return _writeFileSync(getTarget(), fileName, buffer, metadata) + } + + async function writeData(data: Uint8Array, metadata?: LocalMetadata) { + return _writeData(getTarget(), data, metadata) + } + + function writeDataSync(data: Uint8Array, metadata?: LocalMetadata) { + return _writeDataSync(getTarget(), data, metadata) + } + + async function readFile(fileName: string): Promise + async function readFile(fileName: string, encoding: BufferEncoding): Promise + async function readFile(fileName: string, encoding?: BufferEncoding) { + return _readFile(fileName, encoding, maybeGetTarget()) + } + + function open(subkey: string, opt?: OpenOptions) { + return _open(joinSubkey(key, subkey), opt) + } + + function readFileSync(fileName: string): Uint8Array + function readFileSync(fileName: string, encoding: BufferEncoding): string + function readFileSync(fileName: string, encoding?: BufferEncoding) { + return _readFileSync(fileName, encoding, maybeGetTarget()) + } + + async function readData(hash: string) { + return repo.readData(hash) + } + + function readDataSync(hash: string) { + return repo.readDataSync(hash) + } + + async function getMetadata(fileName: string, dataHash?: string) { + return _getMetadata(fileName, dataHash, key) + } + + function getMetadataSync(fileName: string, dataHash?: string) { + return _getMetadataSync(fileName, dataHash, key) + } + + async function copyFile(existingPath: string, newPath: string) { + return _copyFile(getTarget(), existingPath, newPath) + } + + function clear(subkey?: string) { + if (isReadOnly) { + throw new Error(`Cannot clear a store in readonly mode`) + } + + _clear(subkey ? joinSubkey(key, subkey) : key) + } + + function close() { + if (isReadOnly) { + return + } + + // Not saving anything is OK + if (!opened[key]) { + initStore(key) + } + + return _close(key) + } + + return { + // setFile, + writeFile, + writeFileSync, + writeData, + writeDataSync, + readFile, + open, + clear, + close, + fileExistsSync, + readFileSync, + readData, + readDataSync, + mount, + unmount, + dataExistsSync, + getMetadata, + getMetadataSync, + copyFile, + listFiles: async () => { + const f = listFiles() + + return Object.entries(f).map(([k, v]) => ({ name: k, hash: v.hash })) + }, + + // BACKWARDS COMPAT + writeArtifact: (...args: Parameters) => writeData(...args), + writeArtifactSync: (...args: Parameters) => writeDataSync(...args), + readArtifact: (p: string) => readData(getArtifactName(p)), + readArtifactSync: (p: string) => readDataSync(getArtifactName(p)), + resolveArtifact, + + writeData2: (data: any, metadata?: LocalMetadata) => _writeData2(getTarget(), data, metadata), + readData2: (pointer: string) => _readData2(pointer, maybeGetTarget()), + } + } + + async function closeIfOpened(subkey: string) { + const key = joinSubkey('/', subkey) + if (opened[key]) { + return _close(key) + } + } + + const stores = new Map>() + const _open = (key: string, opt?: OpenOptions) => { + if (closed) { + throw new Error(`Artifact store has been closed and cannot be written to`) + } + + if (opt?.readOnly) { + return openStore(key, opt) + } + + if (stores.has(key)) { + return stores.get(key)! + } + + const s = openStore(key, opt) + stores.set(key, s) + + return s + } + + const root = _open('/') + + return { + root, + getPointer, + open: (key: string, opt?: OpenOptions) => !key ? root : root.open(key, opt), + clear: root.clear, + flush, + getStore, + deleteStore, + listStores, + listFiles, + getFilesFromStore, + resolveArtifact, + readFile: root.readFile, + readFileSync: root.readFileSync, + readDataSync, + findFile, + renameStore, + writeFile: root.writeFile, + writeFileSync: root.writeFileSync, + close: () => flush(), + closeIfOpened, + writeArtifact: root.writeArtifact, + writeArtifactSync: root.writeArtifactSync, + readArtifact: root.readArtifact, + readArtifactSync: root.readArtifactSync, + } +} + +function createFileIndex(allFiles: { name: string; hash: string }[]) { + const trie = createTrie, string[]>() + + for (const f of allFiles) { + const parts = f.name.split('/') + const dir = parts.slice(0, -1) + let obj = trie.get(dir) + if (!obj) { + obj = {} + trie.insert(dir, obj) + } + obj[parts.at(-1)!] = f + } + + function readDir(fileName: string) { + const key = fileName.split('/').filter(x => !!x) + const files = trie.get(key) + if (!files) { + return + } + + const res = [ + ...trie.keys(key).map(x => ({ type: 'directory' as const, name: x })), + ...Object.entries(files).map(([k, v]) => ({ type: 'file' as const, name: k, hash: v.hash })), + ] + + return new Map(res.map(x => [x.name, x])) + } + + function statFile(fileName: string) { + const parts = fileName.split('/') + const name = parts.at(-1)! + + if (trie.get(parts)) { + return { type: 'directory' as const, name, size: 0, mtimeMs: 0, ctimeMs: 0 } + } + + const key = parts.slice(0, -1) + const files = trie.get(key) + const stats = files?.[name] + if (!stats) { + // throw Object.assign(new Error(`No such file exists: ${fileName}`), { code: 'ENOENT' }) + return + } + + return { type: 'file' as const, name, hash: stats.hash, size: 0, mtimeMs: 0, ctimeMs: 0 } + } + + return { readDir, statFile } +} + +function resolveFileName(fileName: string, workingDirectory: string) { + const p = path.resolve(workingDirectory, fileName) + const relative = path.relative(workingDirectory, p) + + return { + isInWorkDir: p.startsWith(workingDirectory), + absolute: p, + relative: process.platform === 'win32' ? fixWindowsPath(relative) : relative, + } +} + +export async function toFsFromHash(hash: string, workingDirectory = getWorkingDir(), fs = getFs()): Promise> { + const { index } = await getDataRepository().getBuildFs(hash) + + return toFsFromIndex(index, workingDirectory, fs) +} + +export function toFsFromIndex(buildFsIndex: BuildFsIndex, workingDirectory = getWorkingDir(), fs = getFs()): Fs & SyncFs & Pick { + const fragment = createBuildFsFragment(getDataRepository(), buildFsIndex).root + const wrapped = toFs(workingDirectory, fragment, fs) + + return { + ...wrapped, + readJson: (fileName: string) => wrapped.readFile(fileName.replace(/^\[#([^\[#\]]+)\]/, ''), 'utf-8').then(JSON.parse), + readFile: (fileName: string, encoding?: BufferEncoding) => wrapped.readFile(fileName.replace(/^\[#([^\[#\]]+)\]/, ''), encoding as any), + } as any +} + +export function toFs(workingDirectory: string, buildFs: BuildFsFragment, fs: Fs & SyncFs): Fs & SyncFs { + function resolve(fileName: string) { + return resolveFileName(fileName, workingDirectory) + } + + async function readFile(fileName: string, encoding?: BufferEncoding) { + if (isPointer(fileName)) { + const data = await buildFs.readData(getArtifactName(fileName)) + + return encoding ? Buffer.from(data).toString(encoding) : data + } + + const r = resolve(fileName) + if (buildFs.fileExistsSync(r.relative)) { + return encoding ? buildFs.readFile(r.relative, encoding) : buildFs.readFile(r.relative) + } + + return encoding ? fs.readFile(fileName, encoding) : fs.readFile(fileName) + } + + function readFileSync(fileName: string, encoding?: BufferEncoding) { + if (isPointer(fileName)) { + const data = buildFs.readDataSync(getArtifactName(fileName)) + + return encoding ? Buffer.from(data).toString(encoding) : data + } + + const r = resolve(fileName) + if (buildFs.fileExistsSync(r.relative)) { + return encoding ? buildFs.readFileSync(r.relative, encoding) : buildFs.readFileSync(r.relative) + } + + return encoding ? fs.readFileSync(fileName, encoding) : fs.readFileSync(fileName) + } + + async function writeFile(fileName: string, data: string | Uint8Array, opt?: BufferEncoding | WriteFileOptions) { + if (typeof opt !== 'string' && opt?.fsKey === '#mem') { + return fs.writeFile(fileName, data, opt) + } + + const r = resolve(fileName) + if (r.isInWorkDir) { + const encoding = typeof opt === 'string' ? opt : opt?.encoding + const b = encoding ? Buffer.from(data as string, encoding) : data + + return encoding ? void await buildFs.writeFile(r.relative, b) : void await buildFs.writeFile(r.relative, b) + } + + return fs.writeFile(fileName, data, opt) + } + + function writeFileSync(fileName: string, data: string | Uint8Array, encoding?: BufferEncoding) { + const r = resolve(fileName) + if (r.isInWorkDir) { + const b = encoding ? Buffer.from(data as string, encoding) : data + return encoding ? buildFs.writeFileSync(r.relative, b) : buildFs.writeFileSync(r.relative, b) + } + + return fs.writeFileSync(fileName, data, encoding) + } + + async function deleteFile(fileName: string, opt?: { fsKey?: '#mem' }) { + if (isPointer(fileName)) { + return // store.deleteArtifact(getArtifactName(fileName)) + } + + return fs.deleteFile(fileName, opt) + } + + function deleteFileSync(fileName: string) { + if (isPointer(fileName)) { + return // store.deleteArtifactSync(getArtifactName(fileName)) + } + + return fs.deleteFileSync(fileName) + } + + async function fileExists(fileName: string) { + if (isPointer(fileName)) { + return buildFs.dataExistsSync(getArtifactName(fileName)) + } + + const r = resolve(fileName) + + return buildFs.fileExistsSync(r.relative) || fs.fileExists(fileName) + } + + function fileExistsSync(fileName: string) { + if (isPointer(fileName)) { + return buildFs.dataExistsSync(getArtifactName(fileName)) + } + + const r = resolve(fileName) + + return buildFs.fileExistsSync(r.relative) || fs.fileExistsSync(fileName) + } + + const getFileIndex = memoize(async () => { + const files = await buildFs.listFiles() + + return createFileIndex(files) + }) + + async function readDirectory(fileName: string): Promise { + const r = resolve(fileName) + if (!r.isInWorkDir) { + return fs.readDirectory(fileName) + } + + const index = await getFileIndex() + const actual = await fs.readDirectory(fileName).catch(e => { + throwIfNotFileNotFoundError(e) + return [] + }) + + const res = index.readDir(r.relative) + if (!res) { + return actual + } + + for (const f of actual) { + if (!res.has(f.name)) { + res.set(f.name, f) + } + } + + return [...res.values()] + } + + async function stat(fileName: string): Promise { + const r = resolve(fileName) + if (!r.isInWorkDir) { + return fs.stat(fileName) + } + + const index = await getFileIndex() + const stats = index.statFile(r.relative) + + return stats ?? fs.stat(fileName) + } + + return { + link: fs.link, + stat, + readDirectory, + writeFile, + writeFileSync, + deleteFile, + deleteFileSync, + fileExists, + fileExistsSync, + readFile: readFile as Fs['readFile'], + readFileSync: readFileSync as SyncFs['readFileSync'], + } +} + +// type JsonArtifactSchema> = 'tree' | { +// [P in keyof T]+?: T[P] extends Record ? JsonArtifactSchema | boolean : boolean +// } + +// interface JsonArtifactMapping { +// readonly data: any +// readonly fields?: Record +// } + +// async function serializeJsonArtifact(store: OpenedArtifactStore, data: any, schema: JsonArtifactSchema | true): Promise { +// if (schema === true || typeof data !== 'object' || !data || Array.isArray(data)) { +// return { +// data: await store.writeArtifact(Buffer.from(JSON.stringify(data), 'utf-8')), +// } +// } + +// const serialized: Promise[] = [] +// if (schema === 'tree') { +// for (const [k, v] of Object.entries(data)) { +// serialized.push(serializeJsonArtifact(store, v, true).then(m => [k, m])) +// } + +// const fields = Object.fromEntries(await Promise.all(serialized)) + +// return { +// data: {}, +// fields, +// } +// } + +// const s: Record = {} +// for (const [k, v] of Object.entries(schema)) { +// const d = data[k] +// if (d === undefined) continue + +// if (!v) { +// s[k] = d +// } else { +// serialized.push(serializeJsonArtifact(store, d, v).then(m => [k, m])) +// } +// } + +// const fields = Object.fromEntries(await Promise.all(serialized)) + +// return { +// data: s, +// fields, +// } +// } + +// async function writeJsonArtifact>(vfs: BuildFs, data: T, schema: JsonArtifactSchema = {}): Promise { +// const mappings: Promise[] = [] +// const serialized: Record = {} +// for (const [k, v] of Object.entries(schema)) { +// const d = data[k] +// if (d === undefined) continue + +// if (v === false) { +// serialized[k] = d +// } else if (v === 'tree') { + +// } else if (typeof d === 'object' && !!d && !Array.isArray(d)) { +// const mapping = writeJsonArtifact(vfs, d, v === true ? undefined : v).then(m => [k, m] as const) +// mappings.push(mapping) +// } else { +// mappings.push(store.writeArtifact(Buffer.from(JSON.stringify(d), 'utf-8')).then(h => [k, h])) +// } +// } + +// return Object.fromEntries(await Promise.all(mappings)) +// } + +interface AsyncObjectBuilder { + addResult(key: U, value: Promise | T[U]): AsyncObjectBuilder + build(): U extends T ? Promise : never +} + +function createAsyncObjectBuilder(): AsyncObjectBuilder { + const result: (Promise<[keyof T, T[keyof T]]> | [keyof T, T[keyof T]])[] = [] + const builder = { addResult, build } as AsyncObjectBuilder + + function addResult(key: U, value: Promise | T[U]) { + if (value instanceof Promise) { + result.push(value.then(v => [key, v])) + } else { + result.push([key, value]) + } + + return builder + } + + async function build() { + return Object.fromEntries(await Promise.all(result)) as T + } + + return builder +} + +async function mapTree(tree: Record, fn: (val: T) => Promise): Promise> { + const result: Promise<[string, U]>[]= [] + for (const [k, v] of Object.entries(tree)) { + result.push(fn(v).then(p => [k, p])) + } + + return Object.fromEntries(await Promise.all(result)) +} + +function mapTreeSync(tree: Record, fn: (val: T) => U): Record { + const result: [string, U][]= [] + for (const [k, v] of Object.entries(tree)) { + result.push([k, fn(v)]) + } + + return Object.fromEntries(result) +} + + +function createStructuredArtifactWriter(key: string, writer: Pick) { + const dependencies: string[] = [] + + async function writeJson(obj: any) { + const data = Buffer.from(JSON.stringify(obj)) + const hash = await writer.writeData(key, data) + dependencies.push(hash) + + return hash + } + + async function writeTree(tree: Record) { + return mapTree(tree, writeJson) + } + + function createBuilder() { + const builder = createAsyncObjectBuilder() + + function _writeJson(key: K, obj: T[K]) { + builder.addResult(key, writeJson(obj) as any) + } + + function _writeTree(key: K, tree: T[K] & Record) { + builder.addResult(key, writeTree(tree) as any) + } + + function _writeRaw(key: K, obj: T[K]) { + builder.addResult(key, obj as any) + } + + return { + writeRaw: _writeRaw, + writeJson: _writeJson, + writeTree: _writeTree, + build: async () => ({ + result: await builder.build(), + dependencies, + }), + } + } + + return { + writeJson, + writeTree, + createBuilder, + } +} + + + +// Artifact dependency chain: +// npm deps (including providers) + source code compilation (per-file per-program) +// -> template + assets (per-program) +// -> deployment artifacts + state (per-process) + +// Tool inputs can also be included in this dependency chain +// Same thing for the source code + +// Installation: +// * Packages +// * Providers +// Compilation: +// * Files +// Synth: +// * Template +// * Assets +// Deploy: +// * State +// * Resource-bound artifact store + +export interface InstallationAttributes { + readonly packageLockTimestamp: number + readonly packages: Record + readonly importMap?: FlatImportMap + readonly mode: 'all' | 'types' | 'none' +} + +interface StatsEntry { + readonly size: number + readonly mtimeMs: number + readonly missing?: boolean + readonly corrupted?: boolean + readonly lastStatTime?: number +} + +export interface DataRepository extends RootArtifactStore { + commitHead(id: string): Promise + getHead(id: string): Promise + putHead(head: Head): Promise + deleteHead(id: string): Promise + statData(hash: string): Promise + hasData(hash: string): Promise + hasDataSync(hash: string): boolean + listHeads(): Promise + serializeBuildFs(buildFs: ReadonlyBuildFs): Promise> + // commit(head: Head['id'], hash: string): Promise + + getDataDir: () => string + getLocksDir: () => string + + getRootBuildFs(id: string): Promise> | ReturnType + getRootBuildFsSync(id: string): ReturnType + copyFrom(repo: DataRepository, buildFs: ReadonlyBuildFs, pack?: boolean): Promise +} + +function getStoreHashes(index: BuildFsIndex) { + const hashes = new Set(Object.values(index.stores).map(s => s.hash)) + for (const f of Object.values(index.files)) { + if (f.storeHash) { + hashes.add(f.storeHash) + } + } + + return hashes +} + + +function isAmbiguousObjectHash(fileName: string) { + if (fileName.length === 64 && fileName.match(/^[0-9a-f]+$/)) { + return true + } + + return false +} + +type BuildFsV2 = ReturnType +function createRootFs(fs: Fs & SyncFs, rootDir: string, defaultId: string, opt?: CreateRootFsOptions) { + const repo = getDataRepository(fs, rootDir) + const changed = new Map() + + const aliases = opt?.aliases + const workingDirectory = opt?.workingDirectory ?? rootDir + + if (aliases && defaultId in aliases) { + defaultId = aliases[defaultId] + } + + function resolveFsId(parsed: { fsId?: string }) { + if (!parsed.fsId) { + return defaultId + } + + if (!aliases) { + return parsed.fsId + } + + return aliases[parsed.fsId] ?? parsed.fsId + } + + interface BaseResolveResult { + readonly fsId: string + readonly storeKey: string + } + + interface PointerResolveResult extends BaseResolveResult { + readonly type: 'pointer' + readonly address: string + } + + interface BuildFsFileResolveResult extends BaseResolveResult { + readonly type: 'bfs-file' + readonly relative: string + readonly absolute: string + } + + interface NormalFileResolveResult { + readonly type: 'file' + readonly absolute: string + } + + type ResolveResult = + | PointerResolveResult + | BuildFsFileResolveResult + | NormalFileResolveResult + + function resolve(fileName: string): ResolveResult { + let fsId = defaultId + let storeKey = '' + + if (hasBuildFsPrefix(fileName)) { + const parsed = parseBuildFsPrefix(fileName) + fsId = resolveFsId(parsed) + fileName = parsed.fileName + storeKey = parsed.storeKey + + // Not 100% sure if this is needed + if (process.platform === 'win32') { + storeKey = storeKey.replaceAll('\\', '/') + } + } + + if (isPointer(fileName)) { + return { + type: 'pointer', + fsId, + storeKey, + address: getArtifactName(fileName) + } + } + + if (isAmbiguousObjectHash(fileName)) { + getLogger().warn(`Found object hash without a scheme: ${fileName}`) + + return { + type: 'pointer', + fsId, + storeKey, + address: fileName, + } + } + + const r = resolveFileName(fileName, workingDirectory) + if (r.isInWorkDir) { + return { + type: 'bfs-file', + fsId, + storeKey, + absolute: r.absolute, + relative: r.relative, + } + } + + return { + type: 'file', + absolute: r.absolute, + } + } + + async function writeData(fileName: string, data: Uint8Array, options?: Omit): Promise { + const r = resolve(fileName) + if (r.type === 'pointer') { + throw new Error(`Not implemented`) + } + + if (r.type === 'file') { + throw new Error(`Not implemented`) + } + + const fs = await repo.getRootBuildFs(r.fsId) + const pointer = await fs.open(r.storeKey).writeData(data, options?.metadata) + // if (r.relative !== '') { + // const arr = targetedWrites[`${r.fsId}:${r.storeKey}:${r.relative}`] ??= [] + // arr.push(pointer) + // } + + return pointer + } + + function writeDataSync(fileName: string, data: Uint8Array, options?: Omit) { + const r = resolve(fileName) + if (r.type === 'pointer') { + throw new Error(`Not implemented`) + } + + if (r.type === 'file') { + throw new Error(`Not implemented`) + } + + if (r.relative !== '') { + throw new Error(`Not implemented`) + } + + const fs = repo.getRootBuildFsSync(r.fsId) + + return fs.open(r.storeKey).writeDataSync(data, options?.metadata) + } + + async function readData(hash: string): Promise { + const bfs = await repo.getRootBuildFs(defaultId) + + return bfs.root.readData(hash) + } + + async function writeFile(fileName: string, data: Uint8Array, options?: Omit): Promise + async function writeFile(fileName: string, data: string, options?: BufferEncoding | WriteFileOptions): Promise + async function writeFile(fileName: string, data: string | Uint8Array, options?: WriteFileOptions): Promise + async function writeFile(fileName: string, data: string | Uint8Array, options?: BufferEncoding | WriteFileOptions): Promise { + const r = resolve(fileName) + if (r.type === 'file') { + return fs.writeFile(r.absolute, data, options) + } + + if (r.type === 'pointer') { + throw new Error(`Cannot write to pointer address`) + } + + const encoding = typeof options === 'string' ? options : options?.encoding + const buf = typeof data === 'string' ? Buffer.from(data, encoding) : data + const opt = typeof options === 'string' ? undefined : options + // const checkChanged = opt?.checkChanged + + const root = await repo.getRootBuildFs(r.fsId) + // const previousFile = checkChanged ? root.findFile(fileName) : undefined + const buildFs = root.open(r.storeKey) + const dependencies = opt?.metadata?.dependencies + + await buildFs.writeFile(r.relative, buf, { ...opt?.metadata, dependencies }) + + // const p = await buildFs.writeFile(r.relative, buf, { ...opt?.metadata, dependencies }) + // if (checkChanged && p.hash !== previousFile?.fileHash) { + // changed.set(fileName, previousFile?.fileHash) + // } + } + + async function readFile(fileName: string, options?: ReadFileOptions & { encoding: undefined }): Promise + async function readFile(fileName: string, options: ReadFileOptions & { encoding: BufferEncoding }): Promise + async function readFile(fileName: string, options: BufferEncoding): Promise + async function readFile(fileName: string, options?: BufferEncoding | ReadFileOptions): Promise { + const encoding = (typeof options === 'string' ? options : options?.encoding) as BufferEncoding + const r = resolve(fileName) + if (r.type === 'file') { + return fs.readFile(r.absolute, encoding) + } + + const buildFs = (await repo.getRootBuildFs(r.fsId)).open(r.storeKey) + + if (r.type === 'pointer') { + const data = await buildFs.readData(r.address) + + return encoding ? decode(data, encoding) : data + } + + if (buildFs.fileExistsSync(r.relative)) { + return buildFs.readFile(r.relative, encoding) + } + + return fs.readFile(r.absolute, encoding) + } + + async function deleteFile(fileName: string, opt?: { fsKey?: '#mem' }): Promise { + const r = resolve(fileName) + if (r.type == 'file') { + return fs.deleteFile(r.absolute, opt) + } + + throw new Error(`Not implemented`) + } + + function deleteFileSync(fileName: string): void { + const r = resolve(fileName) + if (r.type == 'file') { + return fs.deleteFileSync(r.absolute) + } + + throw new Error(`Not implemented`) + } + + function writeFileSync(fileName: string, data: Uint8Array, options?: Omit): void + function writeFileSync(fileName: string, data: string | Uint8Array, options?: WriteFileOptions): void + function writeFileSync(fileName: string, data: string, options?: BufferEncoding | WriteFileOptions): void + function writeFileSync(fileName: string, data: string | Uint8Array, options?: BufferEncoding | WriteFileOptions): void { + const r = resolve(fileName) + if (r.type === 'file') { + return fs.writeFileSync(r.absolute, data, options) + } + + if (r.type === 'pointer') { + throw new Error(`Cannot write to pointer address`) + } + + const encoding = typeof options === 'string' ? options : options?.encoding + const buf = typeof data === 'string' ? Buffer.from(data, encoding) : data + const opt = typeof options === 'string' ? undefined : options + + const buildFs = repo.getRootBuildFsSync(r.fsId).open(r.storeKey) + buildFs.writeFileSync(r.relative, buf, opt?.metadata) + } + + function readFileSync(fileName: string, options?: ReadFileOptions & { encoding: undefined }): Uint8Array + function readFileSync(fileName: string, options: ReadFileOptions & { encoding: BufferEncoding }): string + function readFileSync(fileName: string, options: BufferEncoding): string + function readFileSync(fileName: string, options?: BufferEncoding | ReadFileOptions): string | Uint8Array { + const encoding = (typeof options === 'string' ? options : options?.encoding) as BufferEncoding + const r = resolve(fileName) + if (r.type === 'file') { + return fs.readFileSync(r.absolute, encoding) + } + + const buildFs = repo.getRootBuildFsSync(r.fsId).open(r.storeKey) + + if (r.type === 'pointer') { + const data = buildFs.readDataSync(r.address) + + return encoding ? decode(data, encoding) : data + } + + if (buildFs.fileExistsSync(r.relative)) { + return buildFs.readFileSync(r.relative, encoding) + } + + return fs.readFileSync(r.absolute, encoding) + } + + async function readDirectory(fileName: string): Promise { + const r = resolve(fileName) + if (r.type === 'file') { + return fs.readDirectory(r.absolute) + } + + if (r.type === 'pointer') { + throw new Error(`Not implemented`) + } + + const index = await getFileIndex(r.fsId) + const actual = await fs.readDirectory(r.absolute).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + return [] + }) + + const res = index.readDir(r.relative) + if (!res) { + return actual + } + + for (const f of actual) { + if (!res.has(f.name)) { + res.set(f.name, f) + } + } + + return [...res.values()] + } + + async function listStores(fileName: string): Promise<{ name: string }[]> { + const r = resolve(fileName) + if (r.type === 'file') { + throw new Error(`Not implemented`) + } + + if (r.type === 'pointer' || r.relative !== '') { + throw new Error(`Not implemented`) + } + + const fs = await repo.getRootBuildFs(r.fsId) + + return Object.keys(fs.listStores()).map(k => ({ name: k })) + } + + async function rename(currentFileName: string, newFileName: string): Promise { + const r1 = resolve(currentFileName) + const r2 = resolve(newFileName) + if (r1.type === 'file' || r2.type === 'file') { + throw new Error(`Not implemented`) + } + + if (r1.type === 'pointer' || r1.relative !== '' || r2.type === 'pointer' || r2.relative === '') { + throw new Error(`Not implemented`) + } + + if (r1.fsId !== r2.fsId) { + throw new Error(`Not implemented: ${r1.fsId} !== ${r2.fsId}`) + } + + const fs = await repo.getRootBuildFs(r1.fsId) + fs.renameStore(r1.storeKey, r2.storeKey) + } + + async function fileExists(fileName: string) { + const r = resolve(fileName) + if (r.type === 'file') { + return fs.fileExists(r.absolute) + } + + const buildFs = (await repo.getRootBuildFs(r.fsId)).open(r.storeKey) + + if (r.type === 'pointer') { + return buildFs.dataExistsSync(r.address) + } + + return buildFs.fileExistsSync(r.relative) || fs.fileExists(r.absolute) + } + + function fileExistsSync(fileName: string) { + const r = resolve(fileName) + if (r.type === 'file') { + return fs.fileExistsSync(r.absolute) + } + + const buildFs = repo.getRootBuildFsSync(r.fsId).open(r.storeKey) + + if (r.type === 'pointer') { + return buildFs.dataExistsSync(r.address) + } + + return buildFs.fileExistsSync(r.relative) || fs.fileExistsSync(r.absolute) + } + + async function link(existingPath: string, newPath: string) { + const r1 = resolve(existingPath) + const r2 = resolve(newPath) + if (r1.type === 'file' && r2.type === 'file') { + return fs.link(r1.absolute, r2.absolute) + } + + // if (r1.type === 'bfs-file' && r2.type === 'bfs-file') { + // if (r1.fsId !== r2.fsId) { + // const b1 = await getBuildFsIndex(r1.fsId) + // const buildFs = (await repo.getRootBuildFs(r2.fsId)).open(r2.storeKey) + // buildFs.mount(`/tmp/${r1.fsId}`, { index: b1, hash: '' }) + // await buildFs.copyFile(`/tmp/${r1.fsId}/${r1.relative}`, r2.relative) + // buildFs.unmount(`/tmp/${r1.fsId}`) + + // } + + // const buildFs = (await repo.getRootBuildFs(r2.fsId)).open(r2.storeKey) + + // return buildFs.copyFile(r1.relative, r2.relative) + // } + + throw new Error(`Not implemented`) + } + + const getFileIndex = keyedMemoize(async (fsId: string) => { + const buildFs = await repo.getRootBuildFs(fsId) + const files = await buildFs.root.listFiles() + + return createFileIndex(files) + }) + + async function stat(fileName: string): Promise { + const r = resolve(fileName) + if (r.type === 'file') { + return fs.stat(r.absolute) + } + + if (r.type === 'pointer') { + return { type: 'file', size: 0, ctimeMs: 0, mtimeMs: 0 } + } + + const index = await getFileIndex(r.fsId) + const stats = index.statFile(r.relative) + + return stats ?? fs.stat(r.absolute) + } + + async function clear(key?: string) { + const buildFs = await repo.getRootBuildFs(defaultId) + buildFs.deleteStore(key ? path.posix.join('/', key) : '/') + + // await flush() + } + + async function writeJson(fileName: string, data: any) { + const [res, pointers, summary] = extractPointers(data) + const dependencies = summary ? Object.entries(summary).flatMap(([k, v]) => v.map(d => `${k}:${d}`)) : undefined + + return writeFile(fileName, JSON.stringify(res), { + metadata: { pointers, dependencies }, + }) + } + + async function readJson(fileName: string): Promise { + const r = resolve(fileName) + if (r.type !== 'bfs-file') { + return readFile(fileName, 'utf-8').then(JSON.parse) + } + + // const buildFs = (await repo.getRootBuildFs(r.fsId)).open(r.storeKey) + const buildFs = await repo.getRootBuildFs(r.fsId) + const pointer = buildFs.getPointer(r.relative) + const [data, metadata] = await Promise.all([ + repo.readData(pointer.hash), + repo.getMetadata2(pointer) + ]) + + const parsed = JSON.parse(decode(data, 'utf-8')) + if (!metadata.pointers) { + return parsed + } + + return applyPointers(parsed, metadata.pointers) + } + + async function getMetadata(fileName: string) { + const r = resolve(fileName) + if (r.type === 'file') { + throw new Error('Not implemented') + } + + if (r.type === 'pointer') { + return repo.getMetadata2(fileName) + } + + const buildFs = (await repo.getRootBuildFs(r.fsId)).open(r.storeKey) + const metadata = await buildFs.getMetadata(r.relative) + + return metadata as ArtifactMetadata & { pointer: string } + } + + const _fs = { + link, + stat, + writeFile, + readFile, + deleteFile, + deleteFileSync, + writeFileSync, + readFileSync, + readDirectory, + fileExists, + fileExistsSync, + } satisfies Fs & SyncFs + + return { + ..._fs, + clear, + writeData, + readData, + writeDataSync, + readJson, + writeJson, + getMetadata, + listStores, + rename, + } +} + +export async function createMountedFs(procId: string, workingDir: string, mounts: Record) { + const repo = getDataRepository() + const bfs = await repo.getRootBuildFs(procId) + for (const [k, v] of Object.entries(mounts)) { + bfs.root.mount(k, v) + } + + return toFs(workingDir, bfs.root, getFs()) +} + +export function createTempMountedFs(index: BuildFsIndex, workingDir: string, mounts?: Record): Fs & SyncFs & { addMounts: (mounts: Record) => void } { + const repo = getDataRepository() + const bfs = createBuildFsFragment(repo, index) + + function addMounts(mounts: Record) { + for (const [k, v] of Object.entries(mounts)) { + bfs.root.mount(k, v) + } + } + + if (mounts) { + addMounts(mounts) + } + + return { + ...toFs(workingDir, bfs.root, getFs()), + addMounts, + } +} + +function sortAndPruneMetadata(manifest: ArtifactStoreManifest) { + function sortInner(metadata: Record) { + for (const [k, v] of Object.entries(metadata)) { + if (v.dependencies) { + const sortedDeps = sortRecord(v.dependencies) + for (const [k2, v2] of Object.entries(sortedDeps)) { + // sortedDeps[k2] = v2.sort((a, b) => a.localeCompare(b)) + sortedDeps[k2] = v2.sort() + } + Object.assign(v, { dependencies: sortedDeps }) + } + } + + return sortRecord(metadata) + } + + if (manifest.type === 'flat') { + return sortInner(manifest.artifacts) + } + + const sorted: DenormalizedStoreManifest['artifacts'] = {} + for (const [k, v] of Object.entries(manifest.artifacts)) { + sorted[k] = sortInner(v) + } + + return sortRecord(sorted) +} + +function isEOFJsonParseError(err: unknown): boolean { + if (err instanceof SyntaxError && err.message === 'Unexpected end of JSON input') { + return true + } + + return false +} + +export async function readJsonRaw(repo: Pick, hash: string): Promise { + try { + return JSON.parse(decode(await repo.readData(hash), 'utf-8')) as T + } catch (e) { + if (isEOFJsonParseError(e)) { + getLogger().warn(`Removing corrupted data: ${hash}`) + await repo.deleteData(hash) + } + + throw e + } +} + +function readJsonRawSync(repo: Pick, hash: string): T { + try { + return JSON.parse(decode(repo.readDataSync(hash), 'utf-8')) as T + } catch (e) { + if (isEOFJsonParseError(e)) { + getLogger().warn(`Removing corrupted data: ${hash}`) + repo.deleteDataSync(hash) + } + + throw e + } +} + +async function writeJsonRaw(repo: Pick, obj: T): Promise { + const data = Buffer.from(JSON.stringify(obj), 'utf-8') + const hash = getHash(data) + await repo.writeData(hash, data) + + return hash +} + +// Merges left to right +// Files from the right indices override files from the left +export function mergeBuilds(builds: BuildFsIndex[]): BuildFsIndex { + const index: BuildFsIndex = { files: {}, stores: {} } + for (const b of builds.reverse()) { + const currentStores = new Set() + for (const [k, v] of Object.entries(b.files)) { + if (index.files[k]) continue + + index.files[k] = v + if (!index.stores[v.store]) { + index.stores[v.store] = b.stores[v.store] + currentStores.add(v.store) + } else if (!v.storeHash && !currentStores.has(v.store)) { + (v as Mutable).storeHash = b.stores[v.store].hash + } + } + } + + return index +} + +// TODO: use this interface +interface PruneBuildParams { + readonly include?: string[] + readonly exclude?: string[] +} + +export function pruneBuild(build: BuildFsIndex, toRemove: string[]): BuildFsIndex { + const index: BuildFsIndex = { files: {}, stores: {} } + const s = new Set(toRemove) + for (const [k, v] of Object.entries(build.files)) { + if (s.has(k)) continue + + index.files[k] = v + index.stores[v.store] ??= build.stores[v.store] + } + + return index +} + +interface CompressedMetadata { + readonly hashes: string[] + readonly data: Record> +} + +function compressMetadata(metadata: Record>) { + const hashes = new Map() + function indexHash(hash: string) { + if (hashes.has(hash)) { + return hashes.get(hash)! + } + + const id = hashes.size + hashes.set(hash, id) + + return id + } + + const data: Record> = {} + for (const [k, v] of Object.entries(metadata)) { + const inner: typeof v = data[indexHash(k)] = {} + for (const [k2, v2] of Object.entries(v)) { + const d = inner[indexHash(k2)] = deepClone(v2) + if (d.dependencies) { + for (const [k3, arr] of Object.entries(d.dependencies)) { + d.dependencies[indexHash(k3)] = arr.map(indexHash) as any as string[] + } + } + } + } + + return { hashes: [...hashes.keys()], data } +} + +interface NormalizedData { + readonly hash: string + readonly metadata: ArtifactMetadata + readonly metadataHash: string +} + +interface NormalizerOptions { + // Removes any non-critical data (e.g. sourcemaps) + readonly strip?: boolean +} + +function createNormalizer(repo: DataRepository, opt?: NormalizerOptions) { + function normalizeWorker(hash: string, metadata: ArtifactMetadata): NormalizedData { + let pointers: Pointers | undefined + const deps: Record> = {} + + if (metadata.pointers) { + function visit(o: any, p: Pointers): Pointers { + if (typeof p === 'string') { + const n = normalize(o, p) + const s = deps[n.metadataHash] ??= new Set() + s.add(n.hash) + + return n.metadataHash + } else if (Array.isArray(p)) { + const r: Pointers[] = [] + for (let i = 0; i < p.length; i++) { + r[i] = visit(o[i], p[i]) + } + + return r + } else if (typeof p === 'object' && !!p) { + const r: Record = {} + for (const [k, v] of Object.entries(p)) { + r[k] = visit(o[k], v) + } + + return r + } + + return p + } + + const data = JSON.parse(decode(repo.readDataSync(hash), 'utf-8')) + pointers = visit(data, metadata.pointers) + } + + if (metadata.dependencies) { + for (const [k, v] of Object.entries(metadata.dependencies)) { + for (const d of v) { + const n = normalize(d, k) + const s = deps[n.metadataHash] ??= new Set() + s.add(n.hash) + } + } + } + + const dependencies = (metadata.dependencies || metadata.pointers) ? {} as Record : undefined + for (const [k, v] of Object.entries(sortRecord(deps))) { + // dependencies![k] = [...v].sort((a, b) => a.localeCompare(b)) + dependencies![k] = [...v].sort() + } + + const sourcemaps = opt?.strip ? undefined : metadata.sourcemaps + + const result = sortRecord({ + ...metadata, + sourcemaps, + pointers, + dependencies, + }) + + const metadataHash = getHash(Buffer.from(JSON.stringify(result), 'utf-8')) + + return { + hash, + metadata: result, + metadataHash + } + } + + const normalized = new Map() + function normalize(hash: string, storeHash: string): NormalizedData { + const key = `${storeHash}:${hash}` + if (normalized.has(key)) { + return normalized.get(key)! + } + + const m = repo.getMetadata(hash, storeHash) + const r = normalizeWorker(hash, m) + normalized.set(key, r) + normalized.set(`${r.metadataHash}:${r.hash}`, r) + + return r + } + + return { normalize } +} + +function createCopier(repo: DataRepository, normalizer = createNormalizer(repo)) { + const copied = new Map() + + async function copyData(hash: string, storeHash: string, sink: (data: Uint8Array, metadata: LocalMetadata) => Promise): Promise { + if (isNullHash(storeHash)) { + const k = `${getEmptyObjectHash()}:${hash}` + const data = await repo.readData(hash) + const p = await sink(data, {}) + copied.set(k, p) + + return p + } + + const n = normalizer.normalize(hash, storeHash) + const k = `${n.metadataHash}:${n.hash}` + if (copied.has(k)) { + return copied.get(k)! + } + + const data = await repo.readData(hash) + + let pointers: Pointers | undefined + const deps = new Set() + if (n.metadata.pointers) { + async function visit(o: any, p: Pointers): Promise { + if (typeof p === 'string') { + const n = await copyData(o, p, sink) + const { storeHash } = n.resolve() + deps.add(n) + + return storeHash + } else if (Array.isArray(p)) { + const r: Pointers[] = [] + for (let i = 0; i < p.length; i++) { + r[i] = await visit(o[i], p[i]) + } + + return r + } else if (typeof p === 'object' && !!p) { + const r: Record = {} + for (const [k, v] of Object.entries(p)) { + r[k] = await visit(o[k], v) + } + + return r + } + + return p + } + + pointers = await visit(JSON.parse(decode(data, 'utf-8')), n.metadata.pointers) + } + + if (n.metadata.dependencies) { + for (const [k, v] of Object.entries(n.metadata.dependencies)) { + for (const d of v) { + const n = await copyData(d, k, sink) + deps.add(n) + } + } + } + + const result = { + ...n.metadata, + pointers, + dependencies: deps.size > 0 ? [...deps] : undefined, + } + + const p = await sink(data, result) + copied.set(k, p) + + return p + } + + function getCopied(p: string) { + const { hash, storeHash } = toDataPointer(p).resolve() + if (isNullHash(storeHash)) { + const k = `${getEmptyObjectHash()}:${hash}` + const r = copied.get(k) + + return r + } + + const n = normalizer.normalize(hash, storeHash) + const k = `${n.metadataHash}:${n.hash}` + + return copied.get(k) + } + + function getCopiedOrThrow(p: string) { + const copied = getCopied(p) + if (!copied) { + throw new Error(`Missing copied pointer: ${p}`) + } + + return copied + } + + return { copyData, getCopiedOrThrow } +} + +export async function consolidateBuild(repo: DataRepository, build: BuildFsIndex, toKeep: string[], opt?: NormalizerOptions) { + const toKeepSet = new Set(toKeep) + const index: BuildFsIndex = { files: {}, stores: {} } + + const copier = opt ? createCopier(repo, createNormalizer(repo, opt)) : undefined + const fragment = createBuildFsFragment(repo, index, copier) + fragment.root.mount('/tmp', { hash: '', index: build }) + + for (const [k, v] of Object.entries(build.files)) { + if (!toKeepSet.has(k)) continue + + await fragment.root.copyFile(`/tmp/${k}`, k) + } + + fragment.root.unmount('/tmp') + fragment.root.close() + + return { copier, index } +} + +export function getPrefixedPath(hash: string) { + return `${hash[0]}${hash[1]}/${hash[2]}${hash[3]}/${hash.slice(4)}` +} + +interface CreateRootFsOptions { + readonly aliases?: Record + readonly workingDirectory?: string +} + +interface PerformanceCounts { + readonly has: Record + readonly read: Record + readonly write: Record +} + +function getRepository(fs: Fs & SyncFs, buildDir: string) { + function getDataDir() { + return path.resolve(buildDir, 'data') + } + + function getHeadsDir() { + return path.resolve(buildDir, 'heads') + } + + function getResolvedDir() { + return path.resolve(buildDir, 'resolved') + } + + function getBlocksDir() { + return path.resolve(buildDir, 'blocks') + } + + function getLocksDir() { + return path.resolve(buildDir, 'locks') + } + + const headDir = getHeadsDir() + const dataDir = getDataDir() + const resolvedDir = getResolvedDir() // For files that need a "flat" name + + const getHeadPath = (id: string) => path.resolve(headDir, id) + const getDataPath = (hash: string) => path.resolve(dataDir, getPrefixedPath(hash)) + const getBlockPath = (hash: string) => path.resolve(getBlocksDir(), hash) + + // PERF + + let bytesWritten = 0 + let bytesSaved = 0 + const sizes: Record = {} + const counts: PerformanceCounts = { has: {}, read: {}, write: {} } + function inc(key: keyof PerformanceCounts, hash: string) { + counts[key][hash] = (counts[key][hash] ?? 0) + 1 + } + + function printCounts() { + const totals = Object.fromEntries( + Object.entries(counts).map(([k, v]) => [k, Object.values(v as Record).reduce((a, b) => a + b, 0)]) + ) as { [P in keyof PerformanceCounts]: number } + + if (Object.values(totals).reduce((a, b) => a + b, 0) < 25) { + return + } + + getLogger().debug(`Artifact repo perf totals -> ${Object.entries(totals).map(([k, v]) => `${k}: ${v}`).join(', ')}`) + getLogger().debug(` Bytes written: ${bytesWritten}`) + getLogger().debug(` Bytes saved: ${bytesSaved}`) + const sortedSizes = Object.entries(sizes).sort((a, b) => b[1] - a[1]) + if (sortedSizes.length > 0) { + getLogger().debug(' ------- TOP 5 HASHES BY SIZE -------') + for (let i = 0; i < Math.min(sortedSizes.length, 5); i++) { + getLogger().debug(` ${sortedSizes[i][0]}: ${sortedSizes[i][1]}`) + } + } + + getLogger().debug(' ------- TOP 5 HASHES PER OP -------') + + for (const [k, v] of Object.entries(counts)) { + const sorted = Object.entries(v as Record).sort((a, b) => b[1] - a[1]) + if (sorted.length === 0) continue + getLogger().debug(` [${k}]`) + for (let i = 0; i < Math.min(sorted.length, 5); i++) { + getLogger().debug(` ${sorted[i][0]}: ${sorted[i][1]}`) + } + } + } + + // process.on('exit', printCounts) + + // END PERF + + const pendingHeadWrites = new Map>() + + function readHead(id: string): Promise { + if (pendingHeadWrites.has(id)) { + return pendingHeadWrites.get(id)! + } + + return fs.readFile(getHeadPath(id), 'utf-8').then(JSON.parse).catch(throwIfNotFileNotFoundError) + } + + function writeHead(id: string, head: Head | undefined): Promise { + if (pendingHeadWrites.has(id)) { + throw new Error(`Concurrent write to head: ${id}`) + } + + const p = head !== undefined + ? fs.writeFile(getHeadPath(id), Buffer.from(JSON.stringify(head), 'utf-8')) + : fs.deleteFile(getHeadPath(id)) + + pendingHeadWrites.set(id, p.then(() => head).finally(() => pendingHeadWrites.delete(id))) + + return p + } + + async function listHeads() { + const heads: Head[] = [] + for (const f of await fs.readDirectory(headDir)) { + if (f.type === 'file') { + const h = await readHead(f.name) + if (h) heads.push(h) + } + } + return heads + } + + async function hasBlock(hash: string) { + inc('has', hash) + + return fs.fileExists(getBlockPath(hash)) + } + + function _writeBlock(hash: string, data: Uint8Array) { + const key = `block:${hash}` + const pending = pendingWrites.get(key) + if (pending) { + return pending.promise + } + + const promise = fs.writeFile(getBlockPath(hash), data) + .finally(() => pendingWrites.delete(key)) + + pendingWrites.set(key, { promise, data }) + + return promise + } + + function writeBlock(hash: string, data: Uint8Array) { + return _writeBlock(hash, data) + } + + async function hasData(hash: string) { + inc('has', hash) + if (isNullHash(hash)) { + return true + } + + return fs.fileExists(getDataPath(hash)) + } + + function hasDataSync(hash: string) { + inc('has', hash) + if (isNullHash(hash)) { + return true + } + + return fs.fileExistsSync(getDataPath(hash)) + } + + const getNullData = memoize(() => Buffer.from(JSON.stringify(null))) + + async function readData(hash: string) { + inc('read', hash) + if (isNullHash(hash)) { + return getNullData() + } + + const p = getDataPath(hash) + + return pendingWrites.get(p)?.data ?? fs.readFile(p) + } + + function readDataSync(hash: string) { + inc('read', hash) + if (isNullHash(hash)) { + return getNullData() + } + + const p = getDataPath(hash) + const d = pendingWrites.get(p)?.data + if (d !== undefined) { + return d + } + + return fs.readFileSync(p) + } + + function throwIfNotFileExistsError(err: unknown): never | void { + if ((err as any).code !== 'EEXIST') { + throw err + } + } + + // Storing the data is needed to support synchronous reads with async writes + const didWrite = new Set([getNullHash()]) + const pendingWrites = new Map, data: Uint8Array }>() + + function _writeData(hash: string, data: Uint8Array, isAsync?: false): void + function _writeData(hash: string, data: Uint8Array, isAsync: true): Promise + function _writeData(hash: string, data: Uint8Array, isAsync?: boolean) { + if (didWrite.has(hash)) { + return isAsync ? Promise.resolve() : void 0 + } + + inc('write', hash) + + function end(e?: unknown) { + didWrite.add(hash) + + if (!e) { + sizes[hash] = data.length + bytesWritten += data.length + } else { + throwIfNotFileExistsError(e) + bytesSaved += data.length + } + } + + const p = getDataPath(hash) + const pending = pendingWrites.get(p) + if (pending) { + if (!isAsync) { + return + } + + return pending.promise + } + + try { + if (!isAsync) { + return fs.writeFileSync(p, data, { flag: 'wx' }) + } + + const promise = fs.writeFile(p, data, { flag: 'wx' }) + .then(end, end) + .finally(() => pendingWrites.delete(p)) + + pendingWrites.set(p, { promise, data }) + + return promise + } catch (e) { + end(e) + } + } + + function writeData(hash: string, data: Uint8Array) { + return _writeData(hash, data, true) + } + + function writeDataSync(hash: string, data: Uint8Array) { + return _writeData(hash, data, false) + } + + // .catch(async e => { + // if ((e as any).message === 'Unexpected end of JSON input') { + // await deleteData(hash) + // } + // throw e + // }) + + async function readManifest(hash: string) { + return await readJsonRaw(root, hash) + } + + function readManifestSync(hash: string) { + return readJsonRawSync(root, hash) + } + + async function getBuildFs(hash: string) { + const index = await _getFsIndex(hash) + + return { hash, index } + } + + const buildFileSystems = new Map | Promise>>() + function getRootBuildFs(id: string) { + if (buildFileSystems.has(id)) { + return buildFileSystems.get(id)! + } + + const s = getBuildFsIndex(id).then(x => { + const y = buildFileSystems.get(id) + if (y && !(y instanceof Promise)) { + return y + } + const fs = createBuildFsFragment(root, x) + buildFileSystems.set(id, fs) + return fs + }) + buildFileSystems.set(id, s) + + return s + } + + function getRootBuildFsSync(id: string) { + const cached = buildFileSystems.get(id) + if (cached && !(cached instanceof Promise)) { + return cached + } + + const index = getBuildFsIndexSync(id) + const fs = createBuildFsFragment(root, index) + buildFileSystems.set(id, fs) + + return fs + } + + const metadata = createReverseIndex() + + function getMetadata(hash: string): Record | undefined + function getMetadata(hash: string, source: string): ArtifactMetadata + function getMetadata(hash: string, source?: string) { + if (!source) { + return metadata.get(hash) + } + + return _getMetadata(hash, source) + } + + const pendingLinks = new Map>() + function doLink(dataPath: string, targetPath: string) { + if (pendingLinks.has(targetPath)) { + return pendingLinks.get(targetPath)! + } + + const p = fs.link(dataPath, targetPath) + .then(() => targetPath) + .catch(e => (throwIfNotFileExistsError(e), targetPath)) + .finally(() => pendingLinks.delete(targetPath)) + + pendingLinks.set(targetPath, p) + + return p + } + + function resolveArtifact(hash: string, opt?: ResolveArtifactOpts) { + const dest = getDataPath(hash) + + // XXX + if (opt?.noWrite) { + if (opt.extname || opt.filePath) { + throw new Error(`Cannot use other options with "noWrite"`) + } + + return dest + } + + if (opt?.name) { + if (opt.extname || opt.filePath) { + throw new Error(`Cannot set 'extname' or 'filePath' with 'name'`) + } + + return doLink(dest, path.resolve(resolvedDir, opt.name)) + } + + // XXX: when resolving we place the artifact in a flat space so we always link + if (!opt?.extname && !opt?.filePath) { + const linked = path.resolve(resolvedDir, hash) + + return doLink(dest, linked) + } + + if (opt.extname && opt.filePath) { + throw new Error(`Cannot set both 'extname' and 'filePath'`) + } + + const linked = opt.extname ? path.resolve(resolvedDir, `${hash}${opt.extname}`) : opt.filePath! + + return doLink(dest, linked) + } + + const pendingDeletes = new Map>() + async function _deleteData(hash: string) { + const p = getDataPath(hash) + const stats = await getStatsFile(p) + delete stats[hash] + await fs.deleteFile(p) + pendingDeletes.delete(hash) + } + + function deleteData(hash: string) { + const p = _deleteData(hash) + pendingDeletes.set(hash, p) + return p + } + + function deleteDataSync(hash: string) { + pendingDeletes.set(hash, _deleteData(hash)) + } + + interface StatsFile { + [hash: string]: StatsEntry + } + + const openedStatFiles = new Map>() + async function saveStats() { + const entries = [...openedStatFiles.entries()] + openedStatFiles.clear() + + await Promise.all(entries.map(async ([k, v]) => fs.writeFile(k, JSON.stringify(await v)))) + } + + function getStatsFile(dataPath: string): Promise | StatsFile { + const fileName = path.resolve(path.dirname(path.dirname(dataPath)), '.stats.json') + if (openedStatFiles.has(fileName)) { + return openedStatFiles.get(fileName)! + } + + const data = tryReadJson(fs, fileName).then(val => { + const f = val ?? {} + openedStatFiles.set(fileName, f) + + return f + }) + openedStatFiles.set(fileName, data) + + return data + } + + async function updateStatsEntry(oldStats: StatsEntry | undefined, hash: string): Promise { + const p = getDataPath(hash) + + try { + const newStats = await fs.stat(p) + const isValidStill = oldStats?.corrupted ? false : oldStats?.mtimeMs === newStats.mtimeMs + const corrupted = isValidStill ? oldStats?.corrupted : isCorrupted(hash, await readData(hash)) + + return { + ...oldStats, + ...newStats, + missing: undefined, + corrupted: corrupted ? true : undefined, + lastStatTime: Date.now(), + } + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + + return { + size: 0, + mtimeMs: 0, + ...oldStats, + missing: true, + lastStatTime: Date.now(), + } + } + } + + async function statData(hash: string) { + if (isNullHash(hash)) { + return { size: 0, mtimeMs: 0 } + } + + const p = getDataPath(hash) + const k = path.basename(p) + const stats = await getStatsFile(p) + const lastStatTime = stats[k]?.lastStatTime + if (!lastStatTime || ((Date.now() - lastStatTime) / 1000 > 300)) { + stats[k] = await updateStatsEntry(stats[k], hash) + } + + return stats[k] + } + + function _getMetadata(hash: string, storeHash: string, isAsync?: false): ArtifactMetadata + function _getMetadata(hash: string, storeHash: string, isAsync: true): Promise | ArtifactMetadata + function _getMetadata(hash: string, storeHash: string, isAsync?: boolean) { + if (isNullHash(storeHash)) { + return {} + } + + const m = metadata.get(hash, storeHash) + if (m) { + return m + } + + const manifest = manifestBacklog[storeHash] + if (manifest) { + delete manifestBacklog[storeHash] + indexManifest(storeHash, manifest) + + return metadata.get(hash, storeHash)! + } + + if (isAsync) { + const p = getStore(storeHash) + if (p instanceof Promise) { + return p.then(s => s.getMetadata(hash, storeHash)) + } + return p.getMetadata(hash, storeHash) + } + + return getStoreSync(storeHash).getMetadata(hash, storeHash) + } + + async function getMetadata2(pointer: string) { + if (!isDataPointer(pointer)) { + throw new Error('Not implemented') + } + + if (isNullMetadataPointer(pointer)) { + return {} + } + + const { hash, storeHash } = pointer.resolve() + + return _getMetadata(hash, storeHash, true) + } + + const root: DataRepository = { + commitStore, + commitHead: commit, + listHeads, + putHead, + getHead: getHeadData, + deleteHead, + readData, + readDataSync, + serializeBuildFs, + getStore: getStore as DataRepository['getStore'], + getStoreSync: getStoreSync as DataRepository['getStoreSync'], + createStore: createStore as DataRepository['createStore'], + getMetadata, + hasData, + hasDataSync, + getBuildFs, + writeData, + writeDataSync, + resolveArtifact, + deleteData, + deleteDataSync, + getDataDir, + getLocksDir, + getMetadata2, + getRootBuildFs, + getRootBuildFsSync, + statData, + copyFrom, + } + + async function saveIndex(index: BuildFsIndex): Promise { + const hash = await writeJsonRaw(root, index) + + return { hash, index } + } + + async function serializeBuildFs2(buildFs: ReadonlyBuildFs, artifacts: Set) { + const objects: [string, string][] = [] + + for (const f of Object.values(buildFs.index.files)) { + const storeHash = f.storeHash ?? buildFs.index.stores[f.store].hash + artifacts.add(f.hash) + artifacts.add(storeHash) + if (!isNullHash(storeHash)) { + objects.push([storeHash, f.hash]) + } + } + + while (objects.length > 0) { + const [storeHash, hash] = objects.pop()! + const store = await getStore(storeHash) + const metadata = store.getMetadata(hash, storeHash) + if (metadata.dependencies) { + for (const [k, v] of Object.entries(metadata.dependencies)) { + for (const d of v) { + if (artifacts.has(d)) continue + if (isNullHash(k)) { + artifacts.add(d) + continue + } + objects.push([k, d]) + artifacts.add(k) + artifacts.add(d) + } + } + } + if (metadata.sourcemaps) { + for (const h of Object.values(metadata.sourcemaps)) { + const d = toDataPointer(h) + const { hash, storeHash } = d.resolve() + artifacts.add(hash) + if (!isNullHash(storeHash)) { + artifacts.add(storeHash) + } + } + } + } + } + + async function serializeBuildFs(buildFs: ReadonlyBuildFs): Promise> { + if (await hasBlock(buildFs.hash)) { + const data = await fs.readFile(getBlockPath(buildFs.hash)) + const block = openBlock(Buffer.isBuffer(data) ? data : Buffer.from(data)) + + return Object.fromEntries(block.listObjects().map(h => [h, block.readObject(h)])) + } + + const stores = getStoreHashes(buildFs.index) + const artifacts = new Set([buildFs.hash]) + + async function visitStore(hash: string, stack = [hash]) { + async function _getStore() { + try { + return await getStore(hash) + } catch (e) { + const trace = stack.map(x => ` ${x}`).join('\n') + getLogger().error('Failed to find object. Trace:\n', trace) + throw e + } + } + + const s = await _getStore() + artifacts.add(hash) + + for (const [k2, v2] of Object.entries(await s.listArtifacts())) { + for (const [k3, v3] of Object.entries(v2)) { + artifacts.add(k3) + + if (v3.dependencies) { + const storePromises: Promise[] = [] + for (const [d, deps] of Object.entries(v3.dependencies)) { + if (isNullHash(d)) { + deps.forEach(x => artifacts.add(x)) + } else if (!artifacts.has(d)) { + artifacts.add(d) + storePromises.push(visitStore(d, [...stack, d])) + } + } + await Promise.all(storePromises) + } + + if (v3.sourcemaps) { + for (const h of Object.values(v3.sourcemaps)) { + const d = toDataPointer(h) + const { hash, storeHash } = d.resolve() + artifacts.add(hash) + if (!isNullHash(storeHash)) { + artifacts.add(storeHash) + } + } + } + } + } + } + + await Promise.all([...stores].map(h => visitStore(h))) + // await serializeBuildFs2(buildFs, artifacts) + + for (const f of Object.values(buildFs.index.files)) { + const storeHash = f.storeHash || buildFs.index.stores[f.store].hash + if (isNullHash(storeHash)) { + artifacts.add(f.hash) + } else if (!stores.has(storeHash)) { // For debug + throw new Error(`Store containing "${f.hash}" was never serialized: ${storeHash}`) + } else if (!artifacts.has(f.hash)) { // For debug + throw new Error(`Object was never serialized from "${storeHash}": ${f.hash}`) + } + } + + const promises: Promise<[string, Uint8Array]>[] = [] + for (const hash of artifacts) { + promises.push(readData(hash).then(d => [hash, d])) + } + + return Object.fromEntries(await Promise.all(promises)) + } + + async function copyFrom(repo: DataRepository, buildFs: ReadonlyBuildFs, pack?: boolean): Promise { + if (repo.getDataDir() === root.getDataDir()) { + return + } + + if ((!pack && await hasData(buildFs.hash)) || (pack && await hasBlock(buildFs.hash))) { + return + } + + const data = await repo.serializeBuildFs(buildFs) + if (!pack) { + // ~150ms + await runTask('objects', 'copyFrom', async () => { + const hasObject = Object.fromEntries( + await Promise.all(Object.keys(data).map(async k => [k, await hasData(k)] as const)) + ) + + await Promise.all(Object.entries(hasObject).filter(x => !x[1]).map(([k]) => writeData(k, data[k]))) + }, 10) + + return + } + + // TODO: if the target repo has a block we can copy it directly + const block = createBlock(Object.entries(data)) + checkBlock(block, buildFs.index, data) // For debugging + + await writeBlock(buildFs.hash, block) + } + + function createStore() { + return createArtifactStore(root, initArtifactStoreState()) as OpenedArtifactStore + } + + async function _openFsBlock(hash: string) { + const data = await fs.readFile(getBlockPath(hash)) + const block = openBlock(Buffer.isBuffer(data) ? data : Buffer.from(data)) + + // XXX: not great + for (const k of block.listObjects()) { + pendingWrites.set(getDataPath(k), { promise: Promise.resolve(), data: block.readObject(k) }) + } + + const index: BuildFsIndex = JSON.parse(block.readObject(hash).toString('utf-8')) + + return index + } + + async function _getFsIndex(hash: string) { + try { + return await _openFsBlock(hash) + } catch (e) { + throwIfNotFileNotFoundError(e) + + return readJsonRaw(root, hash) + } + } + + async function getBuildFsIndex(id: string) { + const head = await getHeadData(id) + const index = head ? await _getFsIndex(head.storeHash) : { stores: {}, files: {} } + + return index + } + + function getBuildFsIndexSync(id: string) { + const head = getHeadDataSync(id) + const index = head ? readJsonRawSync(root, head.storeHash) : { stores: {}, files: {} } + + return index + } + + // Store utils + function getProgramStore(programId: string, workingDirectory: string) { + const openVfs = async (key: string) => { + const vfs = await getRootBuildFs(programId) + + return vfs.open(key) + } + + async function getFullStore() { + const head = await getHeadData(programId) + if (!head?.storeHash) { + // throw new Error(`No program store found`) + return { hash: '', index: { files: {}, stores: {} } } + } + + return getBuildFs(head.storeHash) + } + + async function getPackageStore() { + const head = await getHeadData(`${programId}`) + if (!head?.storeHash) { + // throw new Error(`No program store found`) + return { hash: '', index: { files: {}, stores: {} } } + } + + return getBuildFs(head.storeHash) + } + + // Compilation + + // Synth + async function getSynthStore() { + const vfs = await openVfs('synth') + + async function commitTemplate(template: TfJson) { + await writeTemplate(template, getProgramFs(programId)) + } + + function setDeps(deps: Record) { + for (const [k, v] of Object.entries(deps)) { + vfs.mount(k, v) + } + } + + return { commitTemplate, afs: vfs, setDeps } + } + + async function clear(key: string) { + const r = await getRootBuildFs(`${programId}`) + r.clear(key) + } + + return { clear, getTargets: async () => getTargets(await getRootBuildFs(`${programId}`)), getInstallation, getTemplate: async () => getTemplate(await openVfs('synth')), getSynthStore, getFullStore, getVfs: openVfs, getRoot: () => getRootBuildFs(programId), getModuleMappings, getPackageStore } + } + + // + + function createNullStore(): ClosedArtifactStore { + return { + hash: getNullHash(), + state: 'closed', + listArtifacts: async () => ({}), + listArtifactsSync: () => ({}), + readArtifact: readData, + readArtifactSync: readDataSync, + resolveMetadata: () => { + throw new Error(`Cannot resolve metadata with a null store`) + }, + getMetadata: () => { + throw new Error(`Cannot get metadata with a null store`) + }, + } + } + + const getNullStore = memoize(createNullStore) + + const stores = new Map>() + stores.set(getNullHash(), getNullStore()) + + function getStore(hash: string) { + if (stores.has(hash)) { + return stores.get(hash)! + } + + const s = loadArtifactStore(hash).then(x => { + const y = stores.get(hash) + if (y && !(y instanceof Promise)) { + return y + } + stores.set(hash, x) + return x + }) + stores.set(hash, s) + + return s + } + + function getStoreSync(hash: string) { + const cached = stores.get(hash) + if (cached && !(cached instanceof Promise)) { + return cached + } + + const s = loadArtifactStoreSync(hash) + stores.set(hash, s) + + return s + } + + function commitStore(state: ArtifactStoreState) { + const sorted = sortAndPruneMetadata({ type: state.mode, artifacts: state.workingMetadata } as ArtifactStoreManifest) + const manifest = { + type: state.mode, + artifacts: sorted, + } as ArtifactStoreManifest + + const data = Buffer.from(JSON.stringify(manifest), 'utf-8') + const hash = getHash(data) + writeData(hash, data) + indexManifestLazy(hash, manifest) + + return hash + } + + const manifestBacklog: Record = {} + function indexManifestLazy(hash: string, manifest: ArtifactStoreManifest) { + if (metadata.hasIndex(hash)) { + return + } + + manifestBacklog[hash] = manifest + } + + function indexManifest(hash: string, manifest: ArtifactStoreManifest) { + function hydrateDeps(s: string, o: Record) { + for (const m of Object.values(o)) { + if (m.dependencies?.['']) { + m.dependencies[s] = m.dependencies[''] + delete m.dependencies[''] + } + } + } + + if (metadata.hasIndex(hash)) { + return + } + + if (manifest.type === 'flat') { + hydrateDeps(hash, manifest.artifacts) + metadata.index(hash, { [hash]: manifest.artifacts }) + + return + } + + if (manifest.artifacts['']) { + manifest.artifacts[hash] = manifest.artifacts[''] + delete manifest.artifacts[''] + } + + for (const k of Object.keys(manifest.artifacts)) { + hydrateDeps(k, manifest.artifacts[k]) + } + + metadata.index(hash, manifest.artifacts) + } + + function loadStoreFromManifest(hash: string, manifest: ArtifactStoreManifest) { + indexManifest(hash, manifest) + + return createArtifactStore(root, { + status: 'closed', + hash: hash, + mode: manifest.type === 'flat' ? 'flat' : 'denormalized', + workingMetadata: manifest.artifacts, + } as ArtifactStoreState) as ClosedArtifactStore + } + + async function loadArtifactStore(hash: string) { + const manifest = await readManifest(hash) + + return loadStoreFromManifest(hash, manifest) + } + + function loadArtifactStoreSync(hash: string) { + const manifest = readManifestSync(hash) + + return loadStoreFromManifest(hash, manifest) + } + + async function saveFs(id: string, hash: string) { + const headData = await getHeadData(id) + + const h: Head = { + id, + storeHash: hash, + timestamp: new Date().toISOString(), + previousCommit: headData?.previousCommit, + } + + await putHead(h) + } + + function isRedundantCommit(proposed: Head, previous?: Head) { + if (!previous) { + return false + } + + if (proposed.timestamp === previous.timestamp) { + return true + } + + return ( + proposed.storeHash === previous.storeHash && + proposed.programHash === previous.programHash && + proposed.isRollback === previous.isRollback + ) + } + + async function commit(id: string, programHash?: string, isRollback?: boolean, isTest?: boolean) { + await runTask('flush', id, () => flushFs(id), 1) + + const head = await getHeadData(id) + if (!head) { + return + } + + const previous = head.previousCommit + ? await readJsonRaw(root, head.previousCommit) + : undefined + + const proposed: Head = { ...head, programHash, isRollback, isTest } + if (isRedundantCommit(proposed, previous)) { + return head + } + + const hash = await writeJsonRaw(root, proposed) + const newHead = { ...head, previousCommit: hash } + await putHead(newHead) + + return newHead + } + + async function putHead(head: Head) { + await writeHead(head.id, head) + } + + async function deleteHead(id: string) { + await writeHead(id, undefined).catch(throwIfNotFileNotFoundError) + } + + async function getHeadData(id: string) { + return readHead(id) + } + + function getHeadDataSync(id: string) { + try { + const data = fs.readFileSync(getHeadPath(id), 'utf-8') + + return JSON.parse(data) as Head + } catch (e) { + throwIfNotFileNotFoundError(e) + + return undefined + } + } + + async function printBlockInfo(hash: string) { + const data = await fs.readFile(getBlockPath(hash)) + const buf = Buffer.isBuffer(data) ? data : Buffer.from(data) + const info = getBlockInfo(buf) + + printLine(`Header size: ${info.headerSize}`) + printLine(`Data size: ${info.dataSize}`) + // printLine(`Objects:`) + // for (const h of Object.keys(info.objects)) { + // printLine(h) + // } + + const b = openBlock(buf) + const index: BuildFsIndex = JSON.parse(b.readObject(hash).toString('utf-8')) + checkBlock(buf, index) + + for (const [k, v] of Object.entries(index.files)) { + printLine(`${k}: ${v.hash}`) + } + } + + async function flushFs(id: string) { + let didChange = false + const fs = await buildFileSystems.get(id) + if (!fs) { + return didChange + } + + // XXX: need to do this per-id + await Promise.all([...pendingWrites.values()].map(x => x.promise)) + + const hash = await fs.flush() + if (hash) { + didChange = true + await saveFs(id, hash) + } else { + getLogger().log(`Skipped saving index ${id} (no changes)`) + } + + buildFileSystems.delete(id) + + // The index files may be buffered still + await Promise.all([...pendingWrites.values()].map(x => x.promise)) + + // if (didChange) { + // // This resets any caching that depends on using the fs as the key + // // TODO: add a hash getter to the fs instead to be used as a key + // const prefix = `${buildDir}:${id}` + // for (const k of roots.keys()) { + // if (k.startsWith(prefix)) { + // roots.delete(k) + // } + // } + // } + + return didChange + } + + async function flush() { + await runTask('artifacts', 'flush', async () => { + await Promise.all(pendingDeletes.values()) + await saveStats() + + for (const [k] of buildFileSystems) { + await flushFs(k) + } + + // Possibly needed? + // await Promise.all(pendingHeadWrites.values()) + }, 10) + } + + return { + root, + flush, + commit, + getHead: getHeadData, + putHead, + deleteHead, + getStore, + createStore, + copyFrom, + + getProgramStore, + getRootBuildFs, + getBuildFsIndex, + + saveIndex, + + createRootFs, + + statData, + + // DEBUG + printBlockInfo, + } +} + +const repos: Record> = {} +function getRepo(fs: Fs & SyncFs, rootDir: string) { + const key = `${getExecutionId()}:${rootDir}` + + return repos[key] ??= getRepository(fs, rootDir) +} + +export function getDataRepository(fs: Fs & SyncFs = getFs(), buildDir = getBuildDir()): DataRepository { + return getRepo(fs, buildDir).root +} + +export function getProgramFsIndex(programId: string) { + return getRepo(getFs(), getBuildDir()).getBuildFsIndex(programId) +} + +export function getDeploymentFsIndex(deploymentID: string) { + return getRepo(getFs(), getBuildDir()).getBuildFsIndex(deploymentID) +} + +export async function getInstallation(fs: Pick): Promise { + const pkg = await tryReadJson(fs, 'packages.json') + if (!pkg) { + return + } + + return { packages: pkg?.packages, packageLockTimestamp: pkg?.packageLockTimestamp, importMap: pkg?.flatImportMap, mode: pkg?.mode } +} + +export function checkBlock(block: Buffer, index: BuildFsIndex, source?: Record) { + const b = openBlock(block) + for (const [k, v] of Object.entries(index.files)) { + if (!b.hasObject(v.hash)) { + if (source && !source[v.hash]) { + throw new Error(`Source is missing file: ${k} [${v.hash}]`) + } + throw new Error(`Missing file: ${k} [${v.hash}]`) + } + + const d = b.readObject(v.hash) + if (isCorrupted(v.hash, d)) { + throw new Error(`Corrupted file: ${k}`) + } + } +} + +export async function commitPackages( + fs: Pick, + packages: Record, + flatImportMap: InstallationAttributes['importMap'], + packageLockTimestamp: number, + mode: 'all' | 'types' | 'none' +) { + await fs.writeFile('[#packages]packages.json', JSON.stringify({ + mode, + packages, + flatImportMap, + packageLockTimestamp, + })) +} + +export async function getTargets(fs: Pick): Promise { + return tryReadJson(fs, '[#compile]__targets__.json') +} + +export async function setTargets(fs: Pick, targets: TargetsFile): Promise { + for (const [k, v] of Object.entries(targets)) { + for (const [k2, v2] of Object.entries(v)) { + v[k2] = sortRecord(v2) + } + targets[k] = sortRecord(v) + } + + return await fs.writeFile('[#compile]__targets__.json', JSON.stringify(sortRecord(targets))) +} + +function isSerializedTemplate(template: TfJson | SerializedTemplate): template is SerializedTemplate { + return typeof template['provider'] === 'string' +} + +export async function getTemplate(fs: Pick): Promise { + const rawTemplate = await tryReadJson(fs, 'template.json') + if (!rawTemplate) { + return + } + + if (isSerializedTemplate(rawTemplate)) { + return deserializeTemplate(fs, rawTemplate) + } + + return rawTemplate +} + +export interface TemplateWithHashes { + readonly template: TfJson + readonly hashes: SerializedTemplate +} + +// Mostly useful for quickly finding changes in templates +export async function getTemplateWithHashes(programHash?: string): Promise { + if (!programHash) { + const hashes = await tryReadJson(getProgramFs(), 'template.json') + if (!hashes) { + return + } + + return { + template: isSerializedTemplate(hashes) ? await deserializeTemplate(getProgramFs(), hashes) : hashes, + hashes, + } + } + + const repo = getDataRepository() + const { index } = await repo.getBuildFs(programHash) + const bfs = createBuildFsFragment(repo, index) + + const hashes = await tryReadJson(bfs, 'template.json') + if (!hashes) { + return + } + + return { + template: isSerializedTemplate(hashes) ? await deserializeTemplate(bfs.root, hashes) : hashes, + hashes, + } +} + + +const manifestPath = '[#compile]__runtimeManifest__.json' +const mappingsPath = '[#compile]__infraMapping__.json' + +interface BindingTypes { + readonly name: string + readonly text: string + readonly references?: Record + readonly symbols?: Record + readonly sourcemap?: string +} + +export interface ModuleBindingResult { + readonly id: string + readonly path: string + readonly types?: BindingTypes + readonly internal?: boolean +} + +async function resolveBindingTypes(fs: Pick, types: BindingTypes): Promise { + const [text, sourcemap] = await Promise.all([ + fs.readFile(types.text, 'utf-8'), + types.sourcemap ? fs.readFile(types.sourcemap, 'utf-8') : undefined, + ]) + + return { name: types.name, text, sourcemap, references: types.references, symbols: types.symbols } +} + +async function resolveBindingResult(fs: Pick, result: Omit): Promise { + if (!result.types) { + return result + } + + return { ...result, types: await resolveBindingTypes(fs, result.types) } +} + +export async function readModuleManifest(fs: Pick): Promise | undefined> { + const d = await fs.readJson>(manifestPath).catch(e => { + throwIfNotFileNotFoundError(e) + + return undefined + }) + + if (d && Object.keys(d).length === 0) { + return + } + + return d +} + +export async function writeModuleManifest(fs: JsonFs, manifest: Record) { + await fs.writeJson(manifestPath, manifest) +} + +export async function getModuleMappings(fs: Pick) { + const m = await readModuleManifest(fs) + if (!m) { + return + } + + return Object.fromEntries(Object.values(m).map(v => [v.id, { path: v.path, types: v.types, internal: v.internal }])) +} + +export async function readInfraMappings(fs: Pick): Promise> { + const d = await tryReadJson>(fs, mappingsPath) + + return d ?? {} +} + +export async function writeInfraMappings(fs: Pick, mappings: Record) { + await fs.writeFile(mappingsPath, JSON.stringify(mappings)) +} + + +export async function getPublished(fs: Pick) { + return tryReadJson>(fs, 'published.json') +} + + +function expandResources(resources: Record>) { + const result: Record> = {} + for (const [k, v] of Object.entries(resources)) { + const [type, ...rest] = k.split('.') + const name = rest.join('.') + const group = result[type] ??= {} + group[name] = v + } + return result +} + +function flattenResources(resources: Record>) { + const result: Record = {} + for (const [k, v] of Object.entries(resources)) { + for (const [k2, v2] of Object.entries(v)) { + result[`${k}.${k2}`] = v2 + } + } + return result +} + +const compressTemplates = false + +export async function writeTemplate(template: TfJson, fs: Pick = getProgramFs()): Promise { + if (!compressTemplates) { + await fs.writeFile('template.json', JSON.stringify(template)) + } else { + const { result, dependencies } = await serializeTemplate(fs, template) + await fs.writeFile('template.json', JSON.stringify(result), { metadata: { dependencies: dependencies } }) + } +} + +async function serializeTemplate(fs: Pick, template: TfJson) { + // Serialization could be done automatically but breaking things up too much + // makes compression less efficient + const writer = createStructuredArtifactWriter('[#synth]', fs) + const builder = writer.createBuilder() + builder.writeJson('provider', template.provider) + builder.writeJson('terraform', template.terraform) + + if (template.resource) { + builder.writeTree('resource', flattenResources(template.resource)) + } + + if (template.locals) { + builder.writeTree('locals', template.locals) + } + + if (template.data) { + builder.writeTree('data', flattenResources(template.data)) + } + + if (template.moved) { + builder.writeJson('moved', template.moved) + } + + if (template['//']) { + builder.writeJson('//', template['//']) + } + + return builder.build() +} + +export async function readState(fs: Pick = getDeploymentFs()) { + const rawState = await tryReadJson(fs, '__full-state__.json') ?? (await tryReadJson(fs, 'full-state.json')) + if (!rawState) { + return + } + + return deserializeState(fs, rawState) +} + +async function writeState(fs: Pick, state: TfState) { + const serialized = await serializeState(fs, state) + await fs.writeFile('__full-state__.json', JSON.stringify(serialized.result), { metadata: { dependencies: serialized.dependencies } }) +} + +export async function setResourceProgramHashes(fs: Pick, hashes: Record) { + await fs.writeFile('__resource-program-hashes__.json', JSON.stringify(hashes)) +} + +export async function getResourceProgramHashes(fs: Pick): Promise | undefined> { + return tryReadJson(fs, '__resource-program-hashes__.json') +} + +async function serializeState(fs: Pick, state: TfState) { + const writer = createStructuredArtifactWriter('[#deploy]', fs) + const resources = Object.fromEntries(state.resources.map(r => [`${r.type}.${r.name}`, r])) + const builder = writer.createBuilder<{ resources: typeof resources } & Omit, SerializedState>() + builder.writeRaw('serial', state.serial) + builder.writeRaw('version', state.version) + builder.writeRaw('lineage', state.lineage) + builder.writeTree('resources', resources) + + return builder.build() +} + +async function deserializeState(fs: Pick, state: SerializedState): Promise { + const builder = createAsyncObjectBuilder() + builder.addResult('serial', state.serial) + builder.addResult('version', state.version) + builder.addResult('lineage', state.lineage) + builder.addResult('resources', readTree(state.resources, fs).then(Object.values)) + + return builder.build() +} + +async function readJson(pointer: string, reader: Pick) { + const data = await reader.readData(isPointer(pointer) ? getArtifactName(pointer) : pointer) + + return JSON.parse(Buffer.from(data).toString('utf-8')) as T +} + +async function readTree(tree: Record, reader: Pick) { + return mapTree(tree, x => readJson(x, reader)) +} + +async function deserializeTemplate(reader: Pick, template: SerializedTemplate): Promise { + const builder = createAsyncObjectBuilder() + builder.addResult('provider', readJson(template.provider, reader)) + builder.addResult('terraform', readJson(template.terraform, reader)) + if (template.resource) { + builder.addResult('resource', readTree(template.resource, reader).then(expandResources)) + } + if (template.data) { + builder.addResult('data', readTree(template.data, reader).then(expandResources)) + } + if (template.locals) { + builder.addResult('locals', readTree(template.locals, reader)) + } + if (template.moved) { + builder.addResult('moved', template.moved) + } + if (template['//']) { + builder.addResult('//', readJson(template['//'], reader)) + } + + return builder.build() +} + +export interface Snapshot { + storeHash: string + published?: Record + pointers?: Record> + targets?: TargetsFile + moduleManifest?: Record> + store?: ReadonlyBuildFs + types?: TypesFileData +} + +export function resolveSnapshot(snapshot: Snapshot) { + if (snapshot.pointers) { + for (const [k, v] of Object.entries(snapshot.pointers)) { + for (const [k2, v2] of Object.entries(v)) { + snapshot.pointers[k][k2] = toDataPointer(v2) + } + } + } + + return snapshot +} + +export const getSnapshotPath = (dir: string) => path.resolve(dir, '.synapse', 'snapshot.json') + +async function loadSnapshotFromManifest(repo: DataRepository, fs: Fs & SyncFs, dir: string) { + const manifest = await tryReadJson(fs, getSnapshotPath(dir)) + if (!manifest) { + return + } + + // XXX: only added this check for "snapshot" packages + if (await repo.hasData(manifest.storeHash)) { + const buildFs = await repo.getBuildFs(manifest.storeHash) + + return { + ...resolveSnapshot(manifest), + store: buildFs, + } + } + + // TODO: this breaks when the package is "linked" instead of downloaded + const snapshotRepo = getDataRepository(fs, path.resolve(dir, '.synapse')) + const buildFs = await snapshotRepo.getBuildFs(manifest.storeHash) + await repo.copyFrom(snapshotRepo, buildFs) + + return { + ...resolveSnapshot(manifest), + store: buildFs, + } +} + +const tarballSnapshotFile = '__bfs-snapshot__.json' +export function isSnapshotTarball(tarball: TarballFile[]) { + return tarball[0]?.path === tarballSnapshotFile +} + +function isCorrupted(expectedHash: string, data: Uint8Array) { + return expectedHash !== getHash(data) +} + +export async function writeSnapshotFile(fs: Fs, dir: string, snapshot: Snapshot) { + await fs.writeFile( + path.resolve(dir, '.synapse', 'snapshot.json'), + JSON.stringify(snapshot) + ) +} + +export async function dumpData(dir: string, index: BuildFsIndex, hash: string, pack?: boolean) { + const snapshotRepo = getDataRepository(getFs(), path.resolve(dir, '.synapse')) + await snapshotRepo.copyFrom(getDataRepository(getFs()), { hash, index }, pack) +} + + +export async function unpackSnapshotTarball(repo: DataRepository, tarball: TarballFile[], dest: string) { + const snapshotFile = tarball.shift()! + if (snapshotFile.path !== tarballSnapshotFile) { + throw new Error(`Expected first file of the tarball to be the snapshot manifest, got: ${snapshotFile.path}`) + } + + const snapshot: Snapshot = JSON.parse(decode(snapshotFile.contents, 'utf-8')) + + // Sanity check + if (typeof snapshot.storeHash !== 'string') { + throw new Error(`Unknown snapshot manifest format`) + } + + const pkgRepo = getDataRepository(getFs(), path.resolve(dest, '.synapse')) + for (const f of tarball) { + if (isCorrupted(f.path, f.contents)) { + throw new Error(`Corrupted data: ${f.path}`) + } + + await repo.writeData(f.path, f.contents) + await pkgRepo.writeData(f.path, f.contents) + } + + const hasFsData = await repo.hasData(snapshot.storeHash) + if (!hasFsData) { + throw new Error(`Tarball missing fs index data: ${snapshot.storeHash}`) + } + + const { index } = await repo.getBuildFs(snapshot.storeHash) + await dump(repo, index, dest) + await writeSnapshotFile(getFs(), dest, snapshot) + + return snapshot +} + +// https://w3c.github.io/webappsec/specs/subresourceintegrity/ + +function getDefaultAliases(){ + const bt = getBuildTarget() + if (!bt) { + return {} + } + + const { programId, deploymentId } = bt + const defaultAliases = deploymentId ? { program: programId, process: deploymentId } : { program: programId } + + return defaultAliases as Record +} + +const roots = new Map>() +function getRootFs(primary: string, rootDir: string, workingDirectory?: string) { + const key = `${rootDir}:${primary}:${workingDirectory ?? ''}` + if (roots.has(key)) { + return roots.get(key)! + } + + const defaultAliases = getDefaultAliases() + const f = createRootFs(getFs(), rootDir, primary, { + aliases: defaultAliases, + workingDirectory, + }) + + roots.set(key, f) + + return f +} + +export async function shutdownRepos() { + for (const [k, v] of Object.entries(repos)) { + await v.flush() + delete repos[k] + } +} + +export function getProgramFs(id?: string) { + if (!id) { + const buildDir = getBuildDir() + const workingDirectory = getWorkingDirectory() + const { programId } = getBuildTargetOrThrow() + + return getRootFs(programId, buildDir, workingDirectory) + } + + const buildDir = getBuildDir(id) + const workingDirectory = getWorkingDir(id) + + return getRootFs(id, buildDir, workingDirectory) +} + +export function getDeploymentFs(id?: string, programId?: string, projectId?: string) { + if (!id) { + const { deploymentId } = getBuildTargetOrThrow() + const buildDir = getBuildDir() + const workingDirectory = getWorkingDirectory() + if (!deploymentId) { + throw new Error(`No deployment id found`) + } + + return getRootFs(deploymentId, buildDir, workingDirectory) + } + + // FIXME: we need to use both IDs to resolve the build/working dir correctly? + programId ??= getProgramIdFromDeployment(id) + const rootDir = getBuildDir(programId) + const workingDirectory = getWorkingDir(programId, projectId) + + return getRootFs(id, rootDir, workingDirectory) +} + +export async function getPreviousFs(id: string) { + const commits = await listCommits(id, undefined, 1) + const hash = commits[0]?.storeHash + if (!hash) { + return + } + + return toFsFromHash(hash) +} + +export function getPreviousProgramFs(id = getBuildTargetOrThrow().programId) { + return getPreviousFs(id) +} + +export async function didFileMaybeChange(fileName: string, fsId = getBuildTargetOrThrow().programId) { + const commits = await listCommits(fsId, undefined, 1) + const hash = commits[0]?.storeHash + if (!hash) { + return true + } + + const root = await getDataRepository().getRootBuildFs(fsId) + const currentFile = root.findFile(fileName) + const previousIndex = await getDataRepository().getBuildFs(hash) + const previousFile = previousIndex.index.files[fileName] + + return currentFile?.fileHash !== previousFile?.hash +} + +export async function readResourceState(resource: string) { + const { deploymentId } = getBuildTargetOrThrow() + if (!deploymentId) { + throw new Error(`No deployment id available to read resource: ${resource}`) + } + + const deploymentFs = getDeploymentFs(deploymentId) + const { state, deps } = JSON.parse(await deploymentFs.readFile('state.json', 'utf-8')) + const hash = state[resource] + if (!hash) { + throw new Error(`No state found for resource: ${resource}`) + } + + const storeHash = deps[resource] + if (!storeHash) { + throw new Error(`No metadata found for resource: ${resource}`) + } + + const p = createPointer(hash, storeHash) + const m = await deploymentFs.getMetadata(p) + const d = JSON.parse(decode(await deploymentFs.readData(hash), 'utf-8')) + if (!m.pointers) { + return d + } + + return applyPointers(d, m.pointers) +} + +export async function loadSnapshot(location: string): Promise { + const fs = getFs() + const repo = getDataRepository(fs) + const snapshot = await loadSnapshotFromManifest(repo, fs, location) + if (snapshot) { + return snapshot + } + + const res = await resolveProgramBuildTarget(location) + if (!res) { + return + } + + const snapshotRepo = getRepository(fs, res.buildDir) + const programStore = snapshotRepo.getProgramStore(res.programId, res.workingDirectory) + const fullStore = await programStore.getPackageStore() + const deployStore = res.deploymentId ? await snapshotRepo.getBuildFsIndex(res.deploymentId) : undefined + const programFs = getProgramFs(res.programId) + const deploymentFs = res.deploymentId ? getDeploymentFs(res.deploymentId) : undefined + const [targets, moduleMappings, published, pointers, types] = await Promise.all([ + getTargets(programFs), + getModuleMappings(programFs), + deploymentFs ? getPublished(deploymentFs) : undefined, + readPointersFile(programFs), + getTypesFile(programFs), + ]) + + const merged = deployStore + ? { index: mergeBuilds([fullStore.index, deployStore]), hash: 'unknown' } + : fullStore + + return { + published, + targets, + store: merged, + storeHash: merged.hash, + moduleManifest: moduleMappings, + pointers, + types, + } +} + +export async function tryLoadSnapshot(location: string) { + const fs = getFs() + const repo = getDataRepository(fs) + + return await loadSnapshotFromManifest(repo, fs, location) +} + +export async function createSnapshot(fsIndex: BuildFsIndex, programId: string, deploymentId?: string) { + const programFs = getProgramFs(programId) + const deploymentFs = deploymentId ? getDeploymentFs(deploymentId) : undefined + const published = deploymentFs ? await getPublished(deploymentFs) : undefined + const moduleMappings = await getModuleMappings(programFs) + const committed = await getRepo(getFs(), getBuildDir()).saveIndex(fsIndex) + const moduleManifest = moduleMappings ? Object.fromEntries( + await Promise.all( + Object.entries(moduleMappings).map(([k, v]) => resolveBindingResult(programFs, v).then(x => [k, x] as const)) + ) + ) : undefined + + const pointersFile = await readPointersFile(programFs) + const pointers: Record> = {} + for (const [k, v] of Object.entries(pointersFile ?? {})) { + pointers[k] = v + for (const [k2, v2] of Object.entries(v)) { + pointers[k][k2] = isDataPointer(v2) ? toAbsolute(v2) : v2 + } + } + + const snapshot = { + published, + storeHash: committed.hash, + targets: await getTargets(programFs), + pointers, + moduleManifest, + types: await getTypesFile(programFs) + } satisfies Snapshot + + return { snapshot, committed } +} + +export async function linkFs(vfs: BuildFsIndex & { id?: string }, dest: string, clean = false) { + await dump(getDataRepository(), vfs, dest, { clean, writeIndex: true, link: true }) +} + +export async function copyFs(vfs: BuildFsIndex & { id?: string }, dest: string, clean = false, writeIndex = true) { + await dump(getDataRepository(), vfs, dest, { clean, writeIndex, link: false }) +} + +export async function printBlockInfo(hash: string) { + const repo = getRepository(getFs(), getBuildDir()) + await repo.printBlockInfo(hash) +} + + +async function syncHeads(repo: DataRepository, remote: RemoteArtifactRepository, localHead?: Head, remoteHead?: Head) { + if (remoteHead?.storeHash === localHead?.storeHash) { + if (localHead) { + getLogger().debug(`Remote is the same as local, skipping sync`, localHead.id) + } + return + } + + const isLocalNewer = !remoteHead?.timestamp || (localHead && (new Date(remoteHead.timestamp).getTime() < new Date(localHead.timestamp).getTime())) + if (isLocalNewer && localHead) { + getLogger().debug(`Pushing local changes to remote`, localHead.id) + await remote.push(localHead.storeHash) + if (localHead.programHash) { + await remote.push(localHead.programHash) + } + await remote.putHead(localHead) + } else if (remoteHead) { + getLogger().debug(`Pulling remote changes`, remoteHead.id) + await remote.pull(remoteHead.storeHash) + if (remoteHead.programHash) { + await remote.pull(remoteHead.programHash) + } + await repo.putHead(remoteHead) + } +} + +const shouldUseRemote = !!process.env['SYNAPSE_SHOULD_USE_REMOTE'] + +export async function syncRemote(projectId: string, programId: string, deploymentId: string) { + if (!shouldUseRemote) { + return + } + + const repo = getDataRepository() + const remote = createRemoteArtifactRepo(repo, projectId) + + const remoteProcessHead = await remote.getHead(deploymentId) + const localProcessHead = await repo.commitHead(deploymentId) + + await syncHeads(repo, remote, localProcessHead, remoteProcessHead) + + // const remoteProgramHead = await remote.getHead(programId) + // const localProgramHead = await repo.commitHead(programId) + + // await syncHeads(repo, remote, localProgramHead, remoteProgramHead) +} + +/** @deprecated */ +export async function createArtifactFs( + fs: Fs & SyncFs, + buildTarget: BuildTarget, +) { + const programId = buildTarget.programId + const repo = getRepo(fs, buildTarget.buildDir) + + const getCurrentProgramStore = memoize(() => repo.getProgramStore(programId, buildTarget.workingDirectory)) + const getCurrentProcessStore = memoize(async () => { + if (!buildTarget.deploymentId) { + throw new Error(`No process exists for the current build target`) + } + + return getDeploymentStore(buildTarget.deploymentId) + }) + + // XXX: needed for watch mode + async function clearCurrentProgramStore() { + await repo.flush() + getCurrentProgramStore.clear() + } + + + // const remote = createRemoteArtifactRepo(backendClient, repo.root, buildTarget.projectId) + async function saveSnapshot(fsIndex: BuildFsIndex, programId: string, deploymentId?: string) { + const { snapshot, committed } = await createSnapshot(fsIndex, programId, deploymentId) + const serialized = await repo.root.serializeBuildFs(committed) + + const arr = Object.entries(serialized) + arr.unshift([ + '__bfs-snapshot__.json', + Buffer.from(JSON.stringify(snapshot)) + ]) + + return arr + } + + async function readArtifact(hash: string): Promise { + if (isPointer(hash)) { + hash = getArtifactName(hash) + } + + return repo.root.readData(hash) + } + + async function flush() { + //await repo.flush() + } + + + async function commit(state: TfState, programFsHash?: string, isTest?: boolean) { + const deployStore = await getCurrentProcessStore() + await deployStore.commitState(state) + const programStore = await getCurrentProgramStore().getFullStore() + await repo.commit(buildTarget.deploymentId!, programFsHash ?? programStore.hash, !!programFsHash, isTest) + } + + async function getCurrentProgramFsIndex() { + return await getCurrentProgramStore().getFullStore() + } + + async function resetManifest(deploymentId: string) { + await repo.deleteHead(deploymentId) + } + + async function downloadArtifacts(hashes: string[]) { + + } + + // `filePath` is mutually exclusive with `extname` + async function resolveArtifact(pointer: string, opt?: ResolveArtifactOpts): Promise { + return repo.root.resolveArtifact(pointer, opt) + } + + async function copyFs(vfs: BuildFsIndex, dest: string, writeIndex?: boolean) { + await dump(repo.root, vfs, dest, { link: false, writeIndex, clean: true }) + } + + async function getBuildFs(hash: string) { + const { index } = await repo.root.getBuildFs(hash) + + return createBuildFsFragment(repo.root, index) + } + + function getProgramFs2(programHash?: string): ReturnType + function getProgramFs2(programHash: string): Promise + function getProgramFs2(programHash?: string) { + if (!programHash) { + return getProgramFs(programId) + } + + return getBuildFs(programHash).then(r => r.root as BuildFsFragment) + } + + return { + commit, + readArtifact, + resolveArtifact, + + dumpFs, + linkFs, + copyFs, + + getCurrentProgramFsIndex, + + maybeRestoreTemplate, + + listCommits, + + flush, + resetManifest, + + createSnapshot, + loadSnapshot, + saveSnapshot, + + getRootFs, + createStore: repo.createStore, + getCurrentProgramStore, + getCurrentProcessStore, + getProgramFs: getProgramFs2, + getRootBuildFs: repo.getRootBuildFs, + getBuildFsIndex: (id: string) => repo.getBuildFsIndex(id), + + getOverlayedFs, + + repo: repo.root, + + downloadArtifacts, + + clearCurrentProgramStore, + } +} + +export const getArtifactFs = memoize(() => createArtifactFs(getFs(), getBuildTargetOrThrow())) + +export async function dumpFs(id = getBuildTargetOrThrow().programId, dest = path.resolve('.vfs-dump'), repo = getDataRepository()) { + const opt: DumpFsOptions = { clean: true, prettyPrint: true, link: false, writeIndex: true } + + // It's _probably_ a hash + if (id.length === 64) { + const vfs = await repo.getBuildFs(id) + await dump(getDataRepository(), { id: vfs.hash, ...vfs.index }, dest, opt) + + return + } + + function resolveId() { + switch (id) { + case 'program': + return getBuildTargetOrThrow().programId + case 'deployment': + return getBuildTargetOrThrow().deploymentId! + default: + return id + } + } + + const resolved = resolveId() + const head = await repo.getHead(resolved) + if (!head) { + throw new Error(`No build fs found: ${resolved}`) + } + const vfs = await repo.getBuildFs(head.storeHash) + await dump(getDataRepository(), { id: resolved, ...vfs.index }, dest, opt) +} + +// Only used for publishing +export async function getOverlayedFs(workingDirectory: string, index: BuildFsIndex) { + const readOnlyFs = createBuildFsFragment(getDataRepository(), index).root + + return toFs(workingDirectory, readOnlyFs, getFs()) +} + +export async function commitProgram(repo = getDataRepository(), programId = getBuildTargetOrThrow().programId) { + await runTask('commit', programId, () => repo.commitHead(programId), 1) +} + +export type ProcessStore = ReturnType +export function getDeploymentStore(deploymentId: string, repo = getDataRepository()) { + const fs = getDeploymentFs() + + async function getResourceStore(resource: string): Promise { + await init() + + const vfs = await repo.getRootBuildFs(deploymentId) + const key = `/deploy/resource-${resource}` + + return vfs.open(key, { readOnly: true }) + } + + async function createResourceStore(resource: string, dependencies: string[]): Promise { + await init() + + const vfs = await repo.getRootBuildFs(deploymentId) + const key = `/deploy/resource-${resource}` + + return vfs.open(key, { clearPrevious: true }) + } + + const init = memoize(_init) + + async function _init() { + const moved = await getMoved() + if (moved) { + await applyMoved(moved) + } + } + + async function applyMoved(moved: { from: string; to: string }[]): Promise { + const stores = new Set((await fs.listStores('[#deploy]')).map(x => x.name)) + + const csResources = moved.filter(r => r.from.startsWith('synapse_resource.')) + const mapped = Object.fromEntries(csResources.map(r => { + const from = r.from.slice('synapse_resource.'.length) + const to = r.to.slice('synapse_resource.'.length) + + return [from, to] + })) + + const getStoreName = (resource: string) => `[#/deploy/resource-${resource}]` + + const writer = getStateWriter() + for (const [from, to] of Object.entries(mapped)) { + if (stores.has(getStoreName(from))) { + await fs.rename(getStoreName(from), getStoreName(to)) + writer.rename(from, to) + } + } + } + + async function commitState(state: TfState) { + await runTask('artifacts', 'state', async () => { + const csResources = state.resources.filter(r => r.type === 'synapse_resource') + + const fragment = await repo.getRootBuildFs(deploymentId) + const stores = fragment.listStores() + const published: Record = {} + + const rStores = new Set(Array.from(csResources).map(n => `/deploy/resource-${n.name}`)) + for (const [k, v] of Object.entries(stores)) { + if (k.startsWith(`/deploy/resource-`) && !rStores.has(k)) { + fragment.deleteStore(k) + } + } + + for (const [k, v] of Object.entries(fragment.listFiles())) { + if (rStores.has(v.store.name)) { + published[k] = `${v.store.hash}:${v.hash}` + } + } + + await fs.writeFile('[#deploy]published.json', JSON.stringify(published)) + await writeState(fs, state) + }, 10) + } + + const getState = () => { + try { + const text = fs.readFileSync('state.json', 'utf-8') + + return JSON.parse(text) as { + state: Record + deps: Record + } + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } + } + + function isDeployed() { + return getState() !== undefined + } + + interface StateFile { + state: Record + deps: Record + } + + function createStateWriter() { + let state: StateFile | undefined = undefined + + function loadState() { + return state ??= getState() ?? { state: {}, deps: {} } + } + + function saveState(data: StateFile) { + const sorted: StateFile = { + state: sortRecord(data.state), + deps: sortRecord(data.deps), + } + + fs.writeFileSync('[#deploy]state.json', JSON.stringify(sorted), { + metadata: { dependencies: Object.entries(data.state).map(([k, v]) => `${data.deps[k]}:${v}`) } + }) + } + + function update(resource: string, pointer: DataPointer) { + const s = loadState() + const { hash, storeHash } = pointer.resolve() + s.state[resource] = hash + s.deps[resource] = storeHash + + saveState(s) + } + + function remove(resource: string) { + const s = loadState() + delete s.state[resource] + delete s.deps[resource] + + saveState(s) + } + + function rename(from: string, to: string) { + const s = loadState() + const x = s.state[from] + const y = s.deps[from] + delete s.state[from] + delete s.deps[from] + s.state[to] = x + s.deps[to] = y + + saveState(s) + } + + function listResources() { + return Object.keys(loadState().state) + } + + return { update, remove, rename, listResources } + } + + const getStateWriter = memoize(createStateWriter) + + function saveResponse(resource: string, inputDeps: string[], resp: any, opType: 'data' | 'read' | 'create' | 'update' | 'delete') { + const [state, pointers, summary] = extractPointers(resp) + if (opType === 'data') { + return { state, pointers } + } + + if (opType !== 'delete') { + const key = `/deploy/resource-${resource}` + const dependencies = Object.entries(summary ?? {}).map(([k, v]) => v.map(h => `${k}:${h}`)).flat() + const pointer = fs.writeDataSync(`[#${key}]`, Buffer.from(JSON.stringify(state), 'utf-8'), { + metadata: { + pointers, + dependencies: Array.from(new Set([...dependencies, ...inputDeps])), + } + }) + getStateWriter().update(resource, pointer) + } else { + getStateWriter().remove(resource) + } + + return { state, pointers } + } + + return { + getState, + isDeployed, + commitState, + getResourceStore, + createResourceStore, + saveResponse, + } +} + +export async function listCommits(id = getTargetDeploymentIdOrThrow(), repo = getDataRepository(), max = 25) { + const commits: Head[] = [] + const start = await repo.getHead(id) + let head = start + while (commits.length < max) { + if (head?.previousCommit) { + const data = await repo.readData(head.previousCommit).catch(throwIfNotFileNotFoundError) + if (!data) break + + head = JSON.parse(Buffer.from(data).toString('utf-8')) + commits.push(head!) + } else { + break + } + } + + // We don't include the first node if it hasn't been committed yet + // XXX: this is a hack to support rolling back from a pulled commit + // if (start && commits[0] && start.previousCommit !== commits[0].previousCommit && start.programHash) { + // commits.unshift(start) + // } + + return commits +} + +export async function putState(state: TfState, procFs = getDeploymentFs()) { + await writeState(procFs, state) +} + +export async function saveMoved(moved: { from: string; to: string }[]) { + const fs = getProgramFs() + const template: SerializedTemplate | TfJson = await fs.readJson('[#synth]template.json') + await fs.writeJson('[#synth]template.json', { ...template, moved }) +} + +export async function getPreviousDeploymentProgramHash() { + const repo = getDataRepository(getFs()) + const deploymentId = getBuildTargetOrThrow().deploymentId + if (!deploymentId) { + throw new Error(`No process id found`) + } + + const h = await repo.getHead(deploymentId) + if (!h) { + return + } + + const c = h.previousCommit + if (!c) { + return + } + + const previousCommit = await readJsonRaw(repo, c) + const programHash = previousCommit.programHash + if (!programHash) { + throw new Error(`No program found for deployment commit: ${c}`) + } + + return programHash +} + +export async function getMoved(programHash?: string): Promise<{ from: string; to: string }[] | undefined> { + const fs = !programHash ? getProgramFs() : await getFsFromHash(programHash) + const template: SerializedTemplate | TfJson | undefined = await tryReadJson(fs, '[#synth]template.json') + + return template?.moved +} + +export async function getProgramHash(programId = getBuildTargetOrThrow().programId, repo = getDataRepository()) { + const head = await repo.getHead(programId) + + return head?.storeHash +} + +export async function maybeRestoreTemplate(head?: Head) { + const repo = getDataRepository(getFs()) + if (head) { + if (!head.programHash) { + throw new Error(`No program to restore from`) + } + + const { index } = await repo.getBuildFs(head.programHash) + const bfs = createBuildFsFragment(repo, index) + + return getTemplate(bfs.root) + } + + const programHash = await getPreviousDeploymentProgramHash() + if (!programHash) { + return + } + + const fs = await getFsFromHash(programHash, repo) + + return getTemplate(fs) +} + +export async function getFsFromHash(hash: string, repo = getDataRepository()) { + const { index } = await repo.getBuildFs(hash) + const bfs = createBuildFsFragment(repo, index) + + return bfs.root +} + +interface FilePart { + readonly hash: string + readonly chunk: Buffer +} + +async function* createChunkStream(data: Uint8Array | Readable): AsyncIterable { + let cursor = 0 + const buffer = Buffer.alloc(chunkSize) + const inputStream = data instanceof Uint8Array ? Duplex.from(data) : data + + for await (const blob of inputStream) { + for (let i = 0; i < blob.length; i += chunkSize) { + const chunk: Buffer = blob.subarray(i, Math.min(i + chunkSize, blob.length)) + const totalSize = cursor + chunk.length + if (totalSize < chunkSize) { + buffer.set(chunk, cursor) + cursor += chunk.length + } else { + buffer.set(chunk.subarray(0, chunkSize - cursor), cursor) + + yield { + hash: getHash(buffer), + chunk: buffer, + } + + cursor = 0 + } + } + } + + if (cursor > 0) { + const chunk = buffer.subarray(0, cursor) + const hash = getHash(chunk) + + yield { hash, chunk } + } +} + +// function* chunk(arr: [string, T][], size: number) { +// let currentSize = 0 +// let acc: typeof arr = [] + +// for (let i = 0; i < arr.length; i++) { +// const itemSize = arr[i][1] ? arr[i][1]!.content.length : 0 +// if (currentSize + itemSize >= size) { +// if (acc.length > 0) { +// yield acc +// } + +// acc = [] +// currentSize = 0 +// } + +// acc.push(arr[i]) +// currentSize += itemSize +// } + +// if (acc.length > 0) { +// yield acc +// } +// } diff --git a/src/auth/index.ts b/src/auth/index.ts new file mode 100644 index 0000000..c31475e --- /dev/null +++ b/src/auth/index.ts @@ -0,0 +1,407 @@ +import * as path from 'node:path' +import { homedir } from 'node:os' +import { getLogger } from '../logging' +import { type Identity, type Credentials, createIdentityClient } from '@cohesible/auth' +import { getBackendClient } from '../backendClient' +import { getFs } from '../execution' +import { memoize } from '../utils' +import { getUserSynapseDirectory } from '../workspaces' + +export interface StoredCredentials extends Credentials { + readonly expiresAt: number // Epoch ms +} + +// Specific to the current machine +interface AccountConfig { + // Time in minutes + sessionDuration?: number +} + +interface Account extends Identity { + readonly config?: AccountConfig +} + +interface AuthState { + currentAccount?: Account['id'] + readonly accounts: Record +} + +// This is the client-side portion of auth + +export type Auth = ReturnType + +export const getAuth = memoize(() => createAuth()) + +function getClient(): ReturnType { + try { + return createIdentityClient() + } catch { + return getBackendClient() as any + } +} + +export const getAuthClient = getClient + +export function createAuth() { + const fs = getFs() + const client = getClient() + const credsDir = path.resolve(getUserSynapseDirectory(), 'credentials') + const statePath = path.resolve(credsDir, 'state.json') + + async function getAuthState(): Promise { + try { + return JSON.parse(await fs.readFile(statePath, 'utf-8')) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + + return { accounts: {} } + } + } + + async function setAuthState(state: AuthState): Promise { + await fs.writeFile(statePath, JSON.stringify(state, undefined, 4)) + } + + async function putAccount(account: Account, makeCurrent = false) { + const state = await getAuthState() + state.accounts[account.id] = account + if (makeCurrent) { + state.currentAccount = account.id + } + await setAuthState(state) + } + + async function deleteAccount(id: Account['id']) { + const state = await getAuthState() + const account = state.accounts[id] + if (!account) { + return + } + + delete state.accounts[id] + + if (state.currentAccount === id) { + state.currentAccount = undefined + } + + if (account.subtype === 'machine') { + await client.deleteMachineIdentity(account.id) + await fs.deleteFile(getMachineKeyPath(account.id)).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + }) + } + + await fs.deleteFile(path.resolve(credsDir, `${id}.json`)).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + }) + + await setAuthState(state) + } + + async function saveCredentials(identity: Pick, credentials: Credentials) { + const stored: StoredCredentials = { + ...credentials, + // The `- 60` is so we refresh a little earlier than needed + expiresAt: Date.now() + ((credentials.expires_in - 60) * 1000), + } + + await fs.writeFile( + path.resolve(credsDir, `${identity.id}.json`), + JSON.stringify(stored, undefined, 4) + ) + + return stored + } + + async function refreshCredentials(identity: Pick, refreshToken: string) { + const creds = await client.refreshCredentials(refreshToken) + + return saveCredentials(identity, creds) + } + + async function getCredentials(identity: Pick) { + try { + const data = await fs.readFile(path.resolve(credsDir, `${identity.id}.json`), 'utf-8') + const creds: StoredCredentials = JSON.parse(data) + + if (creds.expiresAt <= Date.now()) { + const acc = await getAccount(identity.id) + if (isMachineIdentity(acc)) { + return getMachineCredentials(acc) + } + + if (creds.refresh_token) { + getLogger().debug(`Refreshing credentials for identity "${identity.id}"`) + + return refreshCredentials(identity, creds.refresh_token) + } + + getLogger().debug(`Credentials "${identity.id}" expired without a refresh`) + + return + } + + return creds + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + + const acc = await getAccount(identity.id) + if (isMachineIdentity(acc)) { + return getMachineCredentials(acc) + } + } + } + + async function getMachineCredentials(account: Account & { subtype: 'machine'}) { + const privateKey = await getMachineKey(account) + const creds = await client.getMachineCredentials(account.id, privateKey, account.config?.sessionDuration) + + return saveCredentials(account, creds) + } + + const getMachineKeyPath = (id: string) => path.resolve(getUserSynapseDirectory(), 'machine-identities', id) + + async function getMachineKey(account: Account) { + if (account.subtype !== 'machine') { + throw new Error(`Not a machine identity: ${account.id}`) + } + + const keyPath = getMachineKeyPath(account.id) + + return await fs.readFile(keyPath, 'utf-8') as string + } + + async function createMachineIdentity(makeCurrent?: boolean) { + const identity = await client.createMachineIdentity() + await fs.writeFile(getMachineKeyPath(identity.id), identity.privateKey) + const acc = { ...identity, privateKey: undefined, subtype: 'machine' } as Account & { subtype: 'machine' } + await putAccount(acc, makeCurrent) + + return acc + } + + function isMachineIdentity(account: Account | undefined): account is Account & { subtype: 'machine' } { + return account?.subtype === 'machine' + } + + function mergeAccountConfig(account: Account, config: AccountConfig) { + const merged = { ...account.config, ...config } + if (Object.keys(merged).filter(k => (merged as any)[k] !== undefined).length === 0) { + return { ...account, config: undefined } + } + + return { ...account, config: merged } + } + + async function updateAccountConfig(account: Account, config: AccountConfig) { + const merged = mergeAccountConfig(account, config) + await putAccount(merged) + } + + async function machineLogin() { + const accounts = await listAccounts() + const existingAccount = accounts.find(isMachineIdentity) + if (existingAccount) { + const active = await getActiveAccount() + if (active?.id !== existingAccount.id) { + await setActiveAccount(existingAccount) + } + + await getCredentials(existingAccount) + + return + } + + const account = await createMachineIdentity(true) + await getMachineCredentials(account) + } + + async function listAccounts() { + const state = await getAuthState() + + return Object.values(state.accounts) + } + + async function listIdentityProviders() { + return client.listProviders() + } + + async function getAccount(id: string): Promise { + const state = await getAuthState() + const account = state.accounts[id] + // if (!account) { + // throw new Error('No such account exists') + // } + + return account + } + + async function setActiveAccount(account: Account) { + const state = await getAuthState() + if (!state.accounts[account.id]) { + throw new Error(`No such account exists: ${account.id}`) + } + + state.currentAccount = account.id + await setAuthState(state) + + return account + } + + async function getActiveAccount() { + const state = await getAuthState() + if (!state.currentAccount) { + return + } + + return state.accounts[state.currentAccount] + } + + async function startLogin() { + const type = 'github' + const { providers } = await client.listProviders() + const filtered = providers.filter(p => !type || p.type === type) + if (filtered.length === 0) { + throw new Error(`No identity providers available`) + } + + const providerId = filtered[0].id + const { pollToken, redirectUrl } = await client.startLogin(providerId) + getLogger().log('Open URL to login:', redirectUrl) + + const startTime = Date.now() + while (Date.now() - startTime < 60_000) { + await new Promise(r => setTimeout(r, 2500)) + const token = await client.getToken(providerId, pollToken) + if (token) { + const identity = await client.whoami(token.access_token) + await putAccount(identity, true) + await saveCredentials(identity, token) + + return identity + } + } + + throw new Error(`Timed out waiting for login`) + } + + async function login(target?: string) { + if (!target) { + return startLogin() + } + + const accounts = await listAccounts() + const match = accounts.find(a => a.attributes['username'] === target) + if (!match) { + throw new Error(`No account found with username: ${target}`) + } + + // TODO: refresh credentials + await setActiveAccount(match) + } + + async function logout() { + const state = await getAuthState() + const id = state.currentAccount + if (!id) { + return + } + + await deleteAccount(id) + } + + interface ExportedIdentity { + readonly account: Account + readonly privateKey: string + } + + async function exportIdentity(dest: string) { + const activeAccount = await getActiveAccount() + if (!activeAccount) { + throw new Error(`Nothing to export`) + } + + if (!isMachineIdentity(activeAccount)) { + throw new Error(`Account "${activeAccount.id}" is not a machine identity`) + } + + const data: ExportedIdentity = { + account: activeAccount, + privateKey: await getMachineKey(activeAccount), + } + + await getFs().writeFile(dest, JSON.stringify(data, undefined, 4)) + } + + // Validation is incomplete + async function readExportedIdentity(filePath: string): Promise { + const data = await getInputData(filePath).then(JSON.parse) + if (!data || typeof data !== 'object' || !data.account || !data.privateKey) { + const keys = typeof data === 'object' && !!data ? Object.keys(data) : undefined + throw new Error(`Unknown format: ${typeof data} ${keys ? ` [keys: ${keys.join(', ')}]` : ''}`) + } + + return data + } + + async function getInputData(filePath: string) { + if (filePath !== '-') { + return await getFs().readFile(filePath, 'utf-8') + } + + const chunks: any[] = [] + + return new Promise((resolve, reject) => { + process.stdin.on('data', d => chunks.push(d)) + process.stdin.on('end', () => resolve(chunks.join(''))) + }) + } + + async function importIdentity(filePath: string) { + const exported = await readExportedIdentity(filePath) + const state = await getAuthState() + const id = exported.account.id + if (state.accounts[id]) { + throw new Error(`Already imported account: ${id}`) // TODO: only throw if there's a mis-match somewhere + } + + await fs.writeFile(getMachineKeyPath(id), exported.privateKey) + await putAccount(exported.account, true) + + // Make sure the import worked + const creds = await getCredentials(exported.account) + if (!creds) { + throw new Error(`Missing credentials for identity: ${id}`) + } + + const resp = await client.whoami(creds.access_token) + if (resp.id !== id) { + throw new Error(`Imported identity ID does not match the ID reported by the authentication service`) + } + } + + return { + login, + logout, + getAccount, + getActiveAccount, + getCredentials, + machineLogin, + listAccounts, + listIdentityProviders, + updateAccountConfig, + + // Internal-only for now + exportIdentity, + importIdentity, + } +} + diff --git a/src/backendClient.ts b/src/backendClient.ts new file mode 100644 index 0000000..b3fd6c7 --- /dev/null +++ b/src/backendClient.ts @@ -0,0 +1,115 @@ +import * as secrets from './services/secrets/index' +import type { BackendClient } from './runtime/modules/core' +import { getExecutionId } from './execution' +import { processes } from '@cohesible/resources' +import { readState } from './artifacts' +import { getAuthClient } from './auth' +import { getLogger } from '.' +import { mapResource } from './deploy/deployment' + +type ModulePointer> = T | string + +export type AuthSource = ModulePointer<{ default: (workspace: string, branch?: string) => Credentials | Promise }> +export type LoadedBackendClient = ReturnType + + +function getBootstrapClient() { + const serverAddr = 'http://localhost:8681' + const localClient: BackendClient = { + config: {} as any, + getMissingArtifacts: async () => ({ missing: [] }), + getManifest: async () => ({ artifacts: {} }), + putManifest: async () => {}, + putArtifactsBatch: async () => {}, + getArtifactsBatch: async () => ({}), + getSecret: async (type: string) => { + const envVar = type.toUpperCase().replaceAll('-', '_') + if (process.env[envVar]) { + return { value: process.env[envVar] } + } + + throw new Error(`No secret found: ${type}`) + }, + } as any + + return Object.assign(localClient, { + getState: async (id: string) => { + // XXX: horrible horrible hack + if (id.endsWith('--lib--Bundle')) { + id += '--Closure' + } + + getLogger().log(`Getting state for resource`, id) + const state = await readState() + const r = state?.resources?.find(x => id === `${x.type}.${x.name}`) + const s = r ? mapResource(r)?.state.attributes : undefined + if (!s) { + return + } + + return r!.type === 'synapse_resource' ? s.output.value : s + }, + getToolDownloadUrl: async () => { + return {} as any + }, + getCredentials: async () => ({ + token: '', + expirationEpoch: 0, + }) as any, + config: { + address: serverAddr, + } as any, + }) +} + +function _getClient() { + processes.client.getState + + const identityClient = getAuthClient() + + return { + ...identityClient, + getSecret: secrets.getSecret, + } as any as BackendClient +} + +function _createBackendClient() { + const client = _getClient() + + async function getToolDownloadUrl(type: string, opt?: { os?: string; arch?: string; version?: string }) { + throw new Error('Not implemented') + } + + return Object.assign(client, { + getToolDownloadUrl, + getSecret: (type: string) => { + const envVar = type.toUpperCase().replaceAll('-', '_') + if (process.env[envVar]) { + return { value: process.env[envVar] } + } + + return client.getSecret(type) + } + }) +} + +const clients: Record> = {} +export function getBackendClient() { + const k = getExecutionId() + if (clients[k]) { + return clients[k] + } + + try { + return clients[k] = _createBackendClient() + } catch { + return clients[k] = getBootstrapClient() + } +} + + +interface Credentials { + token: string + expirationEpoch: number +} + diff --git a/src/build-fs/backup.ts b/src/build-fs/backup.ts new file mode 100644 index 0000000..1ab38c2 --- /dev/null +++ b/src/build-fs/backup.ts @@ -0,0 +1,58 @@ +import * as path from 'node:path' +import { DataRepository, getDataRepository, readJsonRaw, Head } from '../artifacts' +import { getFs, runWithContext, throwIfCancelled } from '../execution' +import { createBlock } from './block' +import { collectStats, mergeRepoStats } from './stats' + +export async function createIndexBackup(dest: string) { + const repo = getDataRepository(getFs()) + const indices = new Set() + + async function visitHead(h: Head, source: string) { + indices.add(h.storeHash) + if (h.programHash) { + indices.add(h.programHash) + const { index } = await repo.getBuildFs(h.programHash) + if (index.dependencies) { + Object.values(index.dependencies).forEach(d => indices.add(d)) + } + } + + if (h.previousCommit) { + const commit = await readJsonRaw(repo, h.previousCommit) + await visitHead(commit, source) + } + } + + const stats = await collectStats(repo) + const merged = mergeRepoStats(stats) + + const allObjects = new Set([ + ...merged.objects, + ...merged.stores, + ...merged.indices, + ...merged.commits, + ]) + + const data: Record = {} + for (const h of allObjects) { + data[h] = await repo.readData(h) + } + + const block = createBlock(Object.entries(data)) + await getFs().writeFile(dest, block) + + // for (const h of await repo.listHeads()) { + // await visitHead(h, h.id) + // } + + // // XXX: doing this all in-mem = fast way to OOM + // const blocks = new Map() + // for (const h of indices) { + // const data = await repo.serializeBuildFs(await repo.getBuildFs(h)) + // const block = createBlock(Object.entries(data)) + // blocks.set(h, block) + // } + + // console.log([...blocks.values()].reduce((a, b) => a + b.byteLength, 0) / (1024 * 1024)) +} \ No newline at end of file diff --git a/src/build-fs/block.ts b/src/build-fs/block.ts new file mode 100644 index 0000000..b474a66 --- /dev/null +++ b/src/build-fs/block.ts @@ -0,0 +1,237 @@ + +// import { u8, u32, u64, SizedArray } from '../zig/c/types' + +// // Not as clean as `u8[32]` :/ +// // SizedArray + +// type Sha256Hash = SizedArray + +// interface EncodedBlock { +// readonly numHashes: u32 +// readonly hashes: SizedArray // dependent type on `numHashes` +// // dataSegment +// } + +// interface EncodedObject { +// readonly dataStart: u32 +// // `dataEnd` is found by checking the start of the next object, or the end of the block if there is no next object +// } + + +// Binary block format for build artifacts +// 4-bytes - # of hash LUT entries +// +// +// Data segment + +// # of hashes will always be equal to the # of obj +// Hashes are always sorted, allowing for binary search + +function cmpBuffer(a: Buffer, b: Buffer) { + if (a.byteLength !== b.byteLength) { + return a.byteLength - b.byteLength + } + + if (a.byteLength % 4 !== 0) { + throw new Error(`Not implemented`) + } + + for (let i = 0; i < a.byteLength; i += 4) { + const c = a.readUInt32LE(i) - b.readUInt32LE(i) + if (c !== 0) { + return c + } + } + + return 0 +} + +function cmpBufferWithOffsets(a: Buffer, b: Buffer, c: number, d: number, l: number) { + for (let i = 0; i < l; i += 4) { + const z = a.readUInt32LE(i + c) - b.readUInt32LE(i + d) + if (z !== 0) { + return z + } + } + + return 0 +} + +function cmpTypedArray(a: Uint32Array, b: Uint32Array) { + if (a.length !== b.length) { + return a.length - b.length + } + + for (let i = 0; i < a.length; i += 4) { + const c = a[i] - b[i] + if (c !== 0) { + return c + } + } + + return 0 +} + +const objRecordSize = (32 + 4) + +export function createBlock(objects: [string, Uint8Array][]) { + const bytes = objects.map(o => [Buffer.from(o[0], 'hex'), o[1]] as const) + bytes.sort((a, b) => cmpBuffer(a[0], b[0])) + + const headerSize = 4 + (bytes.length * objRecordSize) + let totalSize = headerSize + for (let i = 0; i < bytes.length; i++) { + totalSize += bytes[i][1].byteLength + } + + const block = Buffer.allocUnsafe(totalSize) + + let cursor = 0 + let dataOffset = 0 + cursor = block.writeUint32LE(bytes.length, cursor) + + for (let i = 0; i < bytes.length; i++) { + const [h, d] = bytes[i] + block.set(h, cursor) + cursor += h.byteLength + cursor = block.writeUint32LE(dataOffset, cursor) + + block.set(d, headerSize + dataOffset) + dataOffset += d.byteLength + } + + return block +} + +export type DataBlock = ReturnType + +export function openBlock(block: Buffer) { + const offsets: Record = {} + const numObjects = block.readUint32LE() + const dataStart = 4 + (numObjects * objRecordSize) + if (block.byteLength <= dataStart) { + throw new Error(`Corrupted block: ${block.byteLength} <= ${dataStart}`) + } + + function binarySearch(b: Buffer) { + let lo = 0 + let hi = numObjects - 1 + while (lo <= hi) { + const m = Math.floor((lo + hi) / 2) + const z = cmpBufferWithOffsets(block, b, 4 + (m * objRecordSize), 0, 32) + if (z < 0) { + lo = m + 1 + } else if (z > 0) { + hi = m - 1 + } else { + return m + } + } + + return -1 + } + + function indexOf(hash: string) { + const o = offsets[hash] + if (o !== undefined) { + return o + } + + const b = Buffer.from(hash, 'hex') + + return offsets[hash] = binarySearch(b) + } + + function hasObject(hash: string) { + return indexOf(hash) !== -1 + } + + function getDataOffset(index: number) { + return block.readUint32LE(4 + (index * objRecordSize) + 32) + dataStart + } + + function readObject(hash: string) { + const index = indexOf(hash) + if (index === -1) { + throw new Error(`Object not found: ${hash}`) + } + + const start = getDataOffset(index) + const end = index === numObjects - 1 ? block.byteLength : getDataOffset(index + 1) + + return block.subarray(start, end) + } + + function listObjects() { + const hashes: string[] = [] + for (let i = 0; i < numObjects; i++) { + const offset = 4 + (i * objRecordSize) + const hash = block.subarray(offset, offset + 32).toString('hex') + offsets[hash] = i + hashes.push(hash) + } + + return hashes + } + + return { readObject, hasObject, listObjects } +} + +export function getBlockInfo(block: Buffer) { + const b = openBlock(block) + const hashes = b.listObjects() + const objects = Object.fromEntries(hashes.map(h => [h, b.readObject(h)])) + const headerSize = 4 + (hashes.length * objRecordSize) + + return { + objects, + headerSize, + dataSize: block.byteLength - headerSize, + } +} + +function isInBloomFilter(key: Buffer, filter: Buffer) { + const size = filter.byteLength + const v = new Uint32Array(key.buffer) + const l = v.length + for (let i = 0; i < l; i += 1) { + const p = v[i] + const index = (p >>> 3) % size + if (!(filter[index] & (1 << (p & 0x7)))) { + return false + } + } + + return true +} + +// The "block" data structure works exceptionally well with bloom +// filters because the object key is a hash of the object +// +// Using 10 bits per-element results in a false-positive rate of ~1% +// +// TODO: 11 bits per-element is theoretically better for 32-byte keys, but is it really? +function createBloomFilterData(keys: Buffer[], bitsPerElement = 10) { + if (keys.length === 0) { + throw new Error(`No keys provided`) + } + + const buf = Buffer.alloc(keys.length * bitsPerElement) + const size = buf.byteLength + for (const k of keys) { + // Assumption: `k` is an array of 4-byte integers + const v = new Uint32Array(k.buffer) + const l = v.length + for (let i = 0; i < l; i += 1) { + const p = v[i] + const index = (p >>> 3) % size + buf[index] |= 1 << (p & 0x7) + } + } + + return buf +} + + +// TODO: add `key` to object metadata and use that (plus `source`?) to delta compress commits +// Named objects can use their names as the key \ No newline at end of file diff --git a/src/build-fs/gc.ts b/src/build-fs/gc.ts new file mode 100644 index 0000000..554ab23 --- /dev/null +++ b/src/build-fs/gc.ts @@ -0,0 +1,374 @@ +import * as path from 'node:path' +import { DataRepository, getDataRepository } from '../artifacts' +import { getFs, runWithContext, throwIfCancelled } from '../execution' +import { ensureDirSync, watchForFile } from '../system' +import { acquireFsLock, getCiType, throwIfNotFileNotFoundError } from '../utils' +import { BuildFsStats, collectStats, diffSets, getEventLogger, mergeRepoStats, printStats } from './stats' +import { colorize, getDisplay, printLine } from '../cli/ui' +import { getLogger } from '..' +import { startGcProcess } from './gcWorker' +import { getBuildDir, getUserSynapseDirectory } from '../workspaces' +import { readKeySync } from '../cli/config' + +const gcIntervalMs = 6 * 60 * 60 * 1000 // 6 hours +export const getGcInfoPath = () => path.resolve(getUserSynapseDirectory(), 'gc-info.json') + +function createGcView() { + const view = getDisplay().getOverlayedView() + const logger = getEventLogger() + + let discovered = 0 + let deleted = 0 + + const status = view.createRow() + function updateStatus() { + const phase = deleted > 0 ? 'destroy' : 'search' + const suffix = phase === 'search' + ? (discovered === 0 ? '' : ` (${discovered})`) + : ` (${deleted} / ${discovered})` + const msg = phase === 'search' ? 'Searching...' : 'Deleting...' + status.update(`${msg}${suffix}`) + } + + logger.onGc(ev => { + if (ev.discovered) { + discovered += ev.discovered + } + if (ev.deleted) { + deleted += ev.deleted + } + updateStatus() + }) + + const printLine = (...args: any[]) => view.writeLine(args.map(String).join(' ')) + + logger.onGcSummary(ev => { + if (!ev.dryrun) { + status.release(colorize('green', `Done!`)) + } else { + status.destroy() + printLine(`Dangling objects`, ev.numObjects) + printLine(`Empty directories (after delete)`, ev.numDirs) + } + }) +} + +const settings = { + maxCommits: 10, + pruneAge: 24 * 60 * 60 * 1000, // 1 day + useLock: false, +} + +function getPruneAge() { + return readKeySync('gc.pruneAge') ?? settings.pruneAge +} + +export function startGarbageCollection(buildDir?: string) { + const ac = new AbortController() + const repo = getDataRepository(getFs(), buildDir) + const cancelError = new Error('Cancelled') + + const locksDir = repo.getLocksDir() + const stopGcPath = path.resolve(locksDir, 'stop-gc') + ensureDirSync(stopGcPath) + + function cancel() { + clearTimeout(cancelTimer) + ac.abort(cancelError) + } + + function cancelOnRequest() { + getLogger().debug(`Received request to stop garbage collection`) + cancel() + } + + function cancelOnTimeout() { + getLogger().debug(`Cancelling garbage collection due to timeout`) + cancel() + } + + const maxGcTime = 60_000 + const cancelTimer = setTimeout(cancelOnTimeout, maxGcTime - 5_000).unref() + + const stopWatcher = watchForFile(stopGcPath) + stopWatcher.onFile(cancelOnRequest) + + /** Returns true on success, false if cancelled */ + async function start() { + await using gcLock = settings.useLock + ? await acquireFsLock(path.resolve(locksDir, 'gc'), maxGcTime) + : undefined + + if (gcLock) { + getLogger().log('Lock acquired') + } + + try { + await cleanArtifacts(repo) + getLogger().log('GC complete') + + return true + } catch (e) { + if (e !== cancelError) { + throw e + } + + return false + } finally { + clearTimeout(cancelTimer) + stopWatcher.dispose() + await getFs().deleteFile(stopGcPath).catch(throwIfNotFileNotFoundError) + } + } + + return runWithContext({ abortSignal: ac.signal }, start) +} + + +interface GcTrigger extends AsyncDisposable { + cancel(): void +} + +// XXX: must be a function, otherwise `Symbol.asyncDispose` won't be initialized +function getAsyncDispose(): typeof Symbol.asyncDispose { + if (!Symbol.asyncDispose) { + const asyncDispose = Symbol.for('Symbol.asyncDispose') + Object.defineProperty(Symbol, 'asyncDispose', { value: asyncDispose, enumerable: true }) + } + + return Symbol.asyncDispose +} + +export function maybeCreateGcTrigger(alwaysRun = false): GcTrigger | undefined { + if (!alwaysRun && getCiType()) { + return + } + + const infoPath = getGcInfoPath() + const statsPromise = getFs().stat(infoPath).catch(e => { + if ((e as any).code === 'ENOENT') { + getFs().writeFile(infoPath, JSON.stringify({})) + } + }) + + let cancelled = false + function cancel() { + cancelled = true + } + + async function shouldRun() { + if (alwaysRun) { + return true + } + + const stats = await statsPromise + if (!stats || cancelled) { + return false + } + + if (!stats.mtimeMs) { + return false // bug + } + + const lastRan = Date.now() - stats.mtimeMs + if (lastRan < gcIntervalMs) { + return false + } + + return true + } + + async function dispose() { + if (!(await shouldRun())) { + return + } + + try { + await startGcProcess(getBuildDir()) + getLogger().log('Started background gc') + } catch (e) { + getLogger().error(`Failed to start gc`, e) + } + } + + return { + cancel, + [getAsyncDispose()]: dispose, + } +} + +async function collectGarbage(repo: DataRepository, exclude: Set, pruneAge = getPruneAge()) { + const fs = getFs() + const emptyDirs = new Set() + const toDelete = new Set() + + function shouldDelete(hash: string): Promise | boolean { + if (exclude.has(hash)) { + return false + } + + if (pruneAge === undefined) { + return true + } + + return repo.statData(hash).then(stats => { + if (stats.missing || stats.corrupted) { + return true + } + + return (Date.now() - stats.mtimeMs) >= pruneAge + }) + } + + const dataDir = repo.getDataDir() + for (const f of await fs.readDirectory(dataDir)) { + if (f.type === 'directory' && f.name.length === 2) { + let isEmpty = true + let discovered = 0 + for (const f2 of await fs.readDirectory(path.resolve(dataDir, f.name))) { + if (f2.type === 'directory' && f2.name.length === 2) { + throwIfCancelled() + + let isEmpty2 = true + for (const f3 of await fs.readDirectory(path.resolve(dataDir, f.name, f2.name))) { + if (f3.type === 'file' && f3.name.length === 60) { + const hash = `${f.name}${f2.name}${f3.name}` + if (await shouldDelete(hash)) { + discovered += 1 + toDelete.add(hash) + } else if (isEmpty2) { + isEmpty2 = false + } + } + } + + if (isEmpty2) { + emptyDirs.add(path.resolve(dataDir, f.name, f2.name)) + } else { + isEmpty = false + } + } + } + + if (isEmpty) { + emptyDirs.add(path.resolve(dataDir, f.name)) + } + + if (discovered) { + getEventLogger().emitGcEvent({ discovered }) + } + } + } + + return { + toDelete, + emptyDirs, + } +} + +export async function cleanArtifacts(repo = getDataRepository(getFs()), dryRun = false, deleteStubs = true, pruneAge?: number) { + if (process.stdout.isTTY) { + createGcView() + } + + async function isStub(v: BuildFsStats) { + if (!(v.objects.size === 1 && v.stores.size === 1 && v.indices.size === 1 && v.commits.size === 0)) { + return false + } + + // Stubs should have `__full-state__.json` as their only file + const { index } = await repo.getBuildFs([...v.indices.values()][0]) + const f = index.files['__full-state__.json'] + + return !!f + } + + const fs = getFs() + const stats = await collectStats(repo, settings.maxCommits) + const stubs = new Set() + + const exclude = new Set() + for (const [k, v] of Object.entries(stats)) { + if (v.objects.size === 0) { + // TODO: mark head for deletion + continue + } + + const { missing = new Set(), corrupted = new Set() } = v + for (const h of [...missing, ...corrupted]) { + exclude.add(h) + } + + throwIfCancelled() + + if (missing.size > 0 || corrupted.size > 0) { + printLine(`ID: ${k} [missing: ${missing.size}; corrupted: ${corrupted.size}]`) + } else if (dryRun) { + // await printStats(k, v) + } + + for (const h of corrupted) { + await repo.deleteData(h) + } + + if (deleteStubs && await isStub(v)) { + v.objects.clear() + v.indices.clear() + v.stores.clear() + stubs.add(k) + } + } + + const merged = mergeRepoStats(stats) + + const allObjects = new Set([ + ...merged.objects, + ...merged.stores, + ...merged.indices, + ...merged.commits, + ]) + + getLogger().log(`Total objects found`, allObjects.size) + + const garbage = await collectGarbage(repo, allObjects, pruneAge) + if (dryRun) { + getEventLogger().emitGcSummaryEvent({ + numObjects: garbage.toDelete.size, + numDirs: garbage.emptyDirs.size, + dryrun: true, + }) + + return + } + + if (garbage.toDelete.size > 0) { + getLogger().log(`Deleting ${garbage.toDelete.size} objects`) + } else { + getLogger().log(`Nothing to delete`) + } + + for (const h of garbage.toDelete) { + throwIfCancelled() + + await repo.deleteData(h) + getEventLogger().emitGcEvent({ deleted: 1 }) + } + + // Empty dirs need to be deleted longest-path first + const dirs = [...garbage.emptyDirs].sort((a, b) => b.length - a.length) + for (const d of dirs) { + throwIfCancelled() + + await fs.deleteFile(path.resolve(d, '.stats.json')).catch(throwIfNotFileNotFoundError) + await fs.deleteFile(d, { recursive: false, force: false }) + + for (const s of stubs) { + await repo.deleteHead(s) + } + } + + getEventLogger().emitGcSummaryEvent({ + numObjects: garbage.toDelete.size, + numDirs: garbage.emptyDirs.size, + }) +} diff --git a/src/build-fs/gcWorker.ts b/src/build-fs/gcWorker.ts new file mode 100644 index 0000000..f05fadb --- /dev/null +++ b/src/build-fs/gcWorker.ts @@ -0,0 +1,79 @@ +import * as fs from 'fs/promises' +import * as path from 'node:path' +import * as child_process from 'node:child_process' +import { getFs, runWithContext } from '../execution' +import { ensureDir, watchForFile } from '../system' +import { getLogger } from '..' +import { getBuildDir, getLogsDirectory, getUserSynapseDirectory } from '../workspaces' +import { Bundle } from 'synapse:lib' +import { getGcInfoPath, startGarbageCollection } from './gc' +import { logToStderr } from '../cli/logger' + +interface GcInfo {} + +async function runGc() { + const buildDir = process.argv[2] + if (!buildDir) { + throw new Error(`No build dir provided`) + } + + logToStderr(getLogger()) + process.send?.({ status: 'ready' }) + await runWithContext({}, () => startGarbageCollection(buildDir)) + await getFs().writeFile(getGcInfoPath(), JSON.stringify({})) +} + +const startFn = new Bundle(runGc, { + immediatelyInvoke: true, +}) + +const getLogsFile = () => path.resolve(getLogsDirectory(), 'gc.log') + +export async function startGcProcess(buildDir: string) { + const logFile = getLogsFile() + await ensureDir(logFile) + const log = await fs.open(logFile, 'w') + const proc = child_process.fork( + startFn.destination, + [buildDir], + { + stdio: ['ignore', log.fd, log.fd, 'ipc'], + detached: true, + } + ) + + await new Promise((resolve, reject) => { + function onMessage(ev: child_process.Serializable) { + if (typeof ev === 'object' && !!ev && 'status' in ev && ev.status === 'ready') { + close() + } + } + + function onExit(code: number | null, signal: NodeJS.Signals | null) { + if (code) { + close(new Error(`Non-zero exit code: ${code}\n logs: ${logFile}`)) + } else if (signal) { + close(new Error(`Received signal to exit: ${signal}`)) + } + close(new Error(`Process exited without sending a message`)) + } + + function close(err?: any) { + if (err) { + reject(err) + } else { + resolve() + } + proc.removeListener('message', onMessage) + proc.removeListener('error', close) + proc.removeListener('error', onExit) + } + + proc.on('message', onMessage) + proc.on('error', close) + proc.on('exit', onExit) + }).finally(() => log.close()) + + proc.unref() + proc.disconnect() +} diff --git a/src/build-fs/pointers.ts b/src/build-fs/pointers.ts new file mode 100644 index 0000000..daa373c --- /dev/null +++ b/src/build-fs/pointers.ts @@ -0,0 +1,414 @@ +import { getHash, memoize } from '../utils' + +const pointerSymbol = Symbol.for('synapse.pointer') +export const pointerPrefix = 'pointer:' + +export type DataPointer = string & { + readonly ref: string + readonly hash: string; + readonly [pointerSymbol]: true + resolve(): { hash: string; storeHash: string } + isResolved(): boolean + isContainedBy(storeId: string): boolean +} + +export function isDataPointer(h: string): h is DataPointer { + return typeof h === 'object' && !!h && pointerSymbol in h +} + +// This version only beats `startsWith` after millions of calls +function hasPointerPrefix(s: string) { + if (s.charCodeAt(0) !== 112) return false // p + if (s.charCodeAt(1) !== 111) return false // o + if (s.charCodeAt(2) !== 105) return false // i + if (s.charCodeAt(3) !== 110) return false // n + if (s.charCodeAt(4) !== 116) return false // t + if (s.charCodeAt(5) !== 101) return false // e + if (s.charCodeAt(6) !== 114) return false // r + if (s.charCodeAt(7) !== 58) return false // : + + return true +} + +export function createPointer(hash: string, source: { id: string; close: () => string } | string): DataPointer { + let _storeHash: string | undefined + if (typeof source === 'string') { + _storeHash = source + } + + function getStoreHash() { + return _storeHash ??= (source as Exclude).close() + } + + function resolve() { + const storeHash = getStoreHash() + + return { hash, storeHash } + } + + function isResolved() { + return _storeHash !== undefined + } + + function isContainedBy(storeId: string) { + return !!_storeHash ? false : storeId === (source as Exclude).id + } + + // function toString() { + // console.trace('Unexpected implicit coercion to a string') + // return ref + // } + + const ref = `${pointerPrefix}${hash}` + + return Object.assign(ref, { + ref, + hash, + // toString, + resolve, + isResolved, + isContainedBy, + [pointerSymbol]: true as const, + }) +} + +export function toAbsolute(pointer: DataPointer) { + const { hash, storeHash } = pointer.resolve() + + return `${pointerPrefix}${storeHash}:${hash}` +} + +export type TypedDataPointer = DataPointer + +interface ObjectValueMap { + [key: string]: ValueMap +} + +type ArrayValueMap = ValueMap[] + +export type ValueMap = ObjectValueMap | ArrayValueMap | T + +type PointersArray = Pointers[] + +interface PointersObject { + [key: string]: Pointers +} + +export type Pointers = ValueMap + +function extractString(str: string): string | [string, string] { + if (!str.startsWith(pointerPrefix) || str === pointerPrefix) { + return str + } + + const prefixLen = pointerPrefix.length + const len = str.length - prefixLen + if (len === 64) { + return [str.slice(prefixLen), ''] // We're missing the metadata component + } + + const sepIndex = 64 + prefixLen + if (str[sepIndex] !== ':' || len !== 129) { + throw new Error(`Malformed object pointer: ${str}`) + } + + return [str.slice(sepIndex + 1), str.slice(prefixLen, sepIndex)] +} + +export function extractPointers(obj: any): [obj: any, pointers?: Pointers, summary?: Record] { + const summary: Record> = {} + function addToSummary(hash: string, storeHash: string) { + const set = summary[storeHash] ??= new Set() + set.add(hash) + } + + function extractArray(arr: any[]): { value: any[], pointers?: PointersArray } { + const pointers: PointersArray = [] + const value = arr.map((x, i) => { + const r = visit(x) + if (r[1]) { + pointers[i] = r[1] + } + + return r[0] + }) + + return { value, pointers: pointers.length > 0 ? pointers : undefined } + } + + function extractObject(obj: any[]): [obj: any, pointers?: Pointers] { + let didExtract = false + const p: PointersObject = {} + const o: Record = {} + for (const [k, v] of Object.entries(obj)) { + const r = visit(v) + if (r[1]) { + p[k] = r[1] + didExtract = true + } + o[k] = r[0] + } + + if (!didExtract) { + return [o] + } + + return [o, p] + } + + + function visit(obj: any): [obj: any, pointers?: Pointers] { + if (typeof obj === 'string') { + const r = extractString(obj) + if (r === obj) { + return [r] + } + + const s = (r as [string, string])[1] + if (s) { + addToSummary(r[0], s) + } + + // if (keepPrefix) { + // return [`${pointerPrefix}${r[0]}`, r[1]] + // } + + return r as [string, string] + } + + if (typeof obj !== 'object' || obj === null) { + return [obj] + } + + if (isDataPointer(obj)) { + const r = obj.resolve() + addToSummary(r.hash, r.storeHash) + + // if (keepPrefix) { + // return [obj.ref, r.storeHash] + // } + + return [r.hash, r.storeHash] + } + + if (Array.isArray(obj)) { + const r = extractArray(obj) + + return [r.value, r.pointers] + } + + return extractObject(obj) + } + + const r = visit(obj) + + return [r[0], r[1], Object.fromEntries(Object.entries(summary).map(([k, v]) => [k, Array.from(v)]))] +} + +function addMetadataHash(p: string, m: string) { + if (m === '') { + return p.startsWith(pointerPrefix) ? p : `${pointerPrefix}${p}` + } + + if (p.startsWith(pointerPrefix)) { + p = p.slice(pointerPrefix.length) + } + + return createPointer(p, m) +} + +export function applyPointers(obj: any, pointers: Pointers): any { + if (pointers === undefined) { + return obj + } + + if (typeof obj === 'string') { + if (typeof pointers !== 'string') { + throw new Error(`Malformed pointers structure: ${pointers}`) // FIXME + } + + if (pointers === '') { + return `${pointerPrefix}${obj}` + } + + return createPointer(obj, pointers) + } + + if (typeof obj !== 'object' || obj === null) { + return obj + } + + if (Array.isArray(obj)) { + if (!Array.isArray(pointers)) { + throw new Error(`Malformed pointers structure: ${pointers}`) // FIXME + } + + for (let i = 0; i < pointers.length; i++) { + // Emtpy slots can get serialized as `null` + if (pointers[i] === null) { + continue + } + + obj[i] = applyPointers(obj[i], pointers[i]) + } + + return obj + } + + if (typeof pointers !== 'object' || pointers === null) { + throw new Error(`Malformed pointers structure: ${pointers}`) // FIXME + } + + for (const [k, v] of Object.entries(pointers)) { + obj[k] = applyPointers(obj[k], v) + } + + return obj +} + +export function toDataPointer(s: string) { + if (isDataPointer(s)) { + return s + } + + if (!s.startsWith(pointerPrefix)) { + throw new Error(`Not a data pointer: ${s}`) + } + + const [storeHash, hash] = s.slice(pointerPrefix.length).split(':') + if (!hash) { + throw new Error(`Malformed pointer: ${s}`) + } + + return createPointer(hash, storeHash) +} + +function isProbablyPointer(o: any) { + if (typeof o !== 'string') { + return false + } + + if (o.startsWith(pointerPrefix) && o.length >= 64) { + return true + } + + return !!o.match(/[0-9a-f]{64}$/) +} + +export function coerceToPointer(s: any): DataPointer { + if (isDataPointer(s)) { + return s + } + + const startIndex = s.startsWith(pointerPrefix) ? pointerPrefix.length : 0 + const sep = s.indexOf(':', startIndex) + if (sep === -1) { + return createPointer(s.slice(startIndex), getNullHash()) + } + + return createPointer(s.slice(sep + 1), s.slice(startIndex, sep)) +} + +export function maybeConvertToPointer(o: any): any { + if (typeof o !== 'string') { + return o + } + + if (!isProbablyPointer(o)) { + return o + } + + return coerceToPointer(o) +} + +function mapRecord(obj: Record, fn: (value: T, key: string) => U): Record +function mapRecord(obj: Record, fn: (value: T, key: string) => Promise): Promise> +function mapRecord(obj: Record, fn: (value: T, key: string) => Promise | U) { + let isAsync = false + const result: ([string, U] | Promise<[string, U]>)[]= [] + for (const [k, v] of Object.entries(obj)) { + const val = fn(v, k) + if (val instanceof Promise) { + isAsync = true + result.push(val.then(x => [k, x])) + } else { + result.push([k, val]) + } + } + + if (isAsync) { + return Promise.all(result).then(Object.fromEntries) + } + + return Object.fromEntries(result as [string, U][]) +} + +function mapArray(arr: T[], fn: (value: T, index: number) => U): T[] +function mapArray(arr: T[], fn: (value: T, index: number) => Promise): Promise +function mapArray(arr: T[], fn: (value: T, index: number) => Promise | U) { + let isAsync = false + const result: (Promise | U)[] = [] + for (let i = 0; i < arr.length; i++) { + const val = fn(arr[i], i) + isAsync ||= val instanceof Promise + result[i] = val + } + + return isAsync ? Promise.all(result) : result +} + +function mapValueMap(obj: any, mappings: ValueMap, fn: (tag: T, value: any) => U): ValueMap +function mapValueMap(obj: any, mappings: ValueMap, fn: (tag: T, value: any) => Promise): Promise> +function mapValueMap(obj: any, mappings: ValueMap, fn: (tag: T, value: any) => Promise | U): Promise> | ValueMap { + type Ret = Promise> | ValueMap + + function visit(o: any, p: ValueMap): Ret { + if (Array.isArray(p)) { + return mapArray(p, (v, i) => visit(v, o[i])) as Ret + } else if (typeof p === 'object' && !!p) { + return mapRecord(p as Record>, (v, k) => visit(v, o[k])) as Ret + } + + return fn(p, o) + } + + + return visit(obj, mappings) +} + +function mapPointers(obj: any, pointers: Pointers, fn: (storeHash: string, hash: string) => Promise | T) { + return mapValueMap(obj, pointers, fn) +} + +export const getNullHash = memoize(() => getHash(JSON.stringify(null))) +export const getEmptyObjectHash = memoize(() => getHash(JSON.stringify({}))) + +export function isNullHash(hash: string) { + return hash === getNullHash() +} + +export function isNullMetadataPointer(pointer: DataPointer) { + if (!pointer.isResolved()) { + return false + } + + const { storeHash } = pointer.resolve() + + return isNullHash(storeHash) +} + +// Caching the `startsWith` check does appear to be faster +// But there's no easy/fast way to hold a weak ref to a string primitive +// +// +// const f = new Map() +// const isPointer = (s: string) => { +// const o = f.get(s) +// if (o !== undefined) { +// return o +// } + +// const r = s.startsWith('pointer:') +// f.set(s, r) +// return r +// } diff --git a/src/build-fs/remote.ts b/src/build-fs/remote.ts new file mode 100644 index 0000000..943bc30 --- /dev/null +++ b/src/build-fs/remote.ts @@ -0,0 +1,76 @@ +import * as path from 'node:path' +import { DataRepository, Head } from '../artifacts' +import { getLogger } from '../logging' +import { projects } from '@cohesible/resources' +import { getFs } from '../execution' +import { createBlock, openBlock } from './block' +import { getHash, gunzip, gzip } from '../utils' + +export interface RemoteArtifactRepository { + // listHeads(): Promise + getHead(id: string): Promise + putHead(head: Head): Promise + pull(storeHash: string): Promise + push(storeHash: string): Promise +} + +export function createRemoteArtifactRepo( + repo: DataRepository, + projectId: string +): RemoteArtifactRepository { + async function putHead(head: Head) { + await projects.client.putHead(projectId, { + ...head, + indexHash: head.storeHash, + previousCommit: undefined, // XXX + }) + } + + async function getHead(id: Head['id']): Promise { + const head = await projects.client.getHead(projectId, id).catch(e => { + if ((e as any).statusCode !== 404) { + throw e + } + }) + if (!head) { + return + } + + return { + ...head, + storeHash: head.indexHash, + } + } + + async function pull(buildFsHash: string): Promise { + if (await repo.hasData(buildFsHash)) { + getLogger().debug(`Skipped pulling index`, buildFsHash) + return + } + + getLogger().debug(`Pulling index`, buildFsHash) + + const block = await projects.client.getObject(buildFsHash, 'block').then(gunzip) + const b = openBlock(block) + await Promise.all(b.listObjects().map(h => repo.writeData(h, b.readObject(h)))) + } + + async function push(buildFsHash: string): Promise { + getLogger().debug(`Pushing index`, buildFsHash) + + const buildFs = await repo.getBuildFs(buildFsHash) + const data = await repo.serializeBuildFs(buildFs) + const block = createBlock(Object.entries(data)) + const zipped = await gzip(block) + + const hash = getHash(zipped) + await projects.client.putObject(hash, zipped.toString('base64'), 'block', buildFsHash) + } + + return { + getHead, + putHead, + pull, + push, + } +} diff --git a/src/build-fs/stats.ts b/src/build-fs/stats.ts new file mode 100644 index 0000000..ea0a3f5 --- /dev/null +++ b/src/build-fs/stats.ts @@ -0,0 +1,1113 @@ +import { DataRepository, getDataRepository, listCommits, readJsonRaw, Head } from '../artifacts' +import { colorize, printLine, print } from '../cli/ui' +import { getBuildTargetOrThrow, getFs, throwIfCancelled } from '../execution' +import { BaseOutputMessage, getLogger, getTypedEventEmitter } from '../logging' +import { arrayEditDistance, getHash, keyedMemoize, levenshteinDistance, throwIfNotFileNotFoundError } from '../utils' +import { DataPointer, maybeConvertToPointer, isDataPointer, toDataPointer, isNullHash, pointerPrefix, getNullHash } from './pointers' +import { findArtifactByPrefix, getArtifactByPrefix } from './utils' + +export interface BuildFsStats { + readonly objects: Set + readonly stores: Set + readonly indices: Set + readonly commits: Set + + readonly missing?: Set + readonly corrupted?: Set +} + +interface HeadStats extends BuildFsStats { + readonly type: 'program' | 'process' | 'unknown' +} + +type RepoStats = Record + +// Set union +function mergeSets(a: Set, b: Set): Set { + return new Set([...a, ...b]) +} + +function mergeMaps(a: Map, b: Map): Map +function mergeMaps(a: Map, b: Map): Map +function mergeMaps(a: Map, b: Map): Map +function mergeMaps(a: Map, b: Map): Map, V> & Map, U> & Map +function mergeMaps(a: Map, b: Map) { + return new Map([...a.entries(), ...b.entries()]) +} + +function mergeStats(a: BuildFsStats, b: BuildFsStats): BuildFsStats { + return { + // objects: mergeMaps(a.objects, b.objects), + objects: mergeSets(a.objects, b.objects), + stores: mergeSets(a.stores, b.stores), + indices: mergeSets(a.indices, b.indices), + commits: mergeSets(a.commits, b.commits), + } +} + +export function mergeRepoStats(stats: Record) { + let merged = initStats() + for (const [k, v] of Object.entries(stats)) { + merged = mergeStats(merged, v) + } + return merged +} + +// Finds values in `a` but not in `b` aka the relative complement `A \ B` +export function diffSets(a: Set, b: Set): Set { + const c = new Set() + for (const v of a) { + if (!b.has(v)) { + c.add(v) + } + } + return c +} + + +export function initStats(): BuildFsStats { + return { + objects: new Set(), + stores: new Set(), + indices: new Set(), + commits: new Set(), + } +} + +const refTypes = ['head', 'index', 'store', 'commit', 'file', 'object'] as const +type RefType = (typeof refTypes)[number] + +function isRefType(s: string): s is RefType { + return refTypes.includes(s as any) +} + +// `file:${indexHash}:${name}` +// File refs resolve to object refs +type FileRef = `file:${string}:${string}` + +// `object:${storeHash}:${hash}` +type ObjectRef = `object:${string}:${string}` + +// `index:${hash}` +type ContainerRef = `${Exclude}:${string}` + +type Ref = + | FileRef + | ObjectRef + | ContainerRef + + +function _parseRef(ref: Ref) { + const parts = ref.split(':') + const type = parts.shift()! + if (!isRefType(type)) { + throw new Error(`Not a valid ref type: ${type}`) + } + + const firstPart = parts.shift() + if (!firstPart) { + throw new Error(`Invalid ref: ${ref}`) + } + + switch (type) { + case 'file': + case 'object': + const hash = parts.shift() + if (!hash) { + throw new Error(`Invalid ref: ${ref}`) + } + + if (type === 'object') { + return { type, store: firstPart, hash } as const + } + + return { type, index: firstPart, hash } as const + + case 'head': + return { type, id: firstPart } as const + + default: + return { type, hash: firstPart } as const + } +} + +const parseRef = _parseRef + +interface RefInfo { + readonly status: 'ok' | 'missing' | 'corrupted' | 'missing-store' | 'corrupted-store' + readonly ownSize?: number +} + +interface ResolvedRefInfo { + type: RefType, + size?: number + hash?: string + status: RefInfo['status'] +} + +async function getAllRefs(walker: RepoWalker, ref: Ref) { + const refs = new Map() + const queue: Ref[] = [] + + async function enqueue(ref: Ref) { + throwIfCancelled() + + const info = await walker.resolveRef(ref) + refs.set(ref, info) + + if (info.status === 'ok') { + queue.push(ref) + } + } + + await enqueue(ref) + + while (queue.length > 0) { + throwIfCancelled() + + const r = queue.shift()! + const deps = await walker.getDependencies(r) + const p: Promise[] = [] + for (const d of deps) { + if (!refs.has(d)) { + p.push(enqueue(d)) + } + } + await Promise.all(p) + } + + return refs +} + +type RepoWalker = ReturnType +function createRepoWalker(repo: DataRepository, maxCommits?: number) { + const visited = new Set() + const fromRefs = new Map>() + // const toRefs = new Map>() + const commitTtl = new Map() + + const getStats = keyedMemoize(repo.statData) + + function addRef(from: Ref, to: Ref) { + const s1 = fromRefs.get(from) ?? new Set() + //const s2 = toRefs.get(to) ?? new Set() + s1.add(to) + // s2.add(from) + fromRefs.set(from, s1) + // toRefs.set(to, s2) + } + + async function getHashStatus(hash: string): Promise<'ok' | 'missing' | 'corrupted'> { + const stats = await getStats(hash) + + return stats.missing ? 'missing' : stats.corrupted ? 'corrupted' : 'ok' + } + + async function getRefInfo(parsed: ReturnType): Promise { + switch (parsed.type) { + case 'head': + const head = await repo.getHead(parsed.id) + const status = head === undefined ? 'missing' : 'ok' + + return { + status, + ownSize: 0, + } + case 'object': + const storeStatus = await getHashStatus(parsed.store) + if (storeStatus !== 'ok') { + return { status: `${storeStatus}-store` } + } + // Fallsthrough + case 'commit': + case 'index': + case 'store': + const hashStats = await getStats(parsed.hash) + if (hashStats.missing) { + return { status: 'missing' } + } + + return { status: hashStats.corrupted ? 'corrupted' : 'ok', ownSize: hashStats.size } + + default: + throw new Error(`Not implemented: ${parsed.type}`) + } + } + + async function visitObject(hash: string, storeHash: string) { + const refs = new Set() + const m = repo.getMetadata(hash, storeHash) + if (m.sourcemaps) { + for (const [k, v] of Object.entries(m.sourcemaps)) { + // legacy + if (typeof v !== 'string' || !v.startsWith('pointer:')) { + continue + } + + const d = toDataPointer(v) + const { hash, storeHash } = d.resolve() + refs.add(`object:${storeHash}:${hash}` as Ref) + } + } + + if (!m.dependencies) { + return refs + } + + for (const [source, arr] of Object.entries(m.dependencies)) { + for (const d of arr) { + refs.add(`object:${source}:${d}` as Ref) + } + } + + return refs + } + + async function visitStore(hash: string) { + const refs = new Set() + const store = await repo.getStore(hash) + for (const [k, v] of Object.entries(await store.listArtifacts())) { + for (const [k2, v2] of Object.entries(v)) { + refs.add(`object:${k}:${k2}` as Ref) + } + } + + return refs + } + + const getFsIndex = keyedMemoize(repo.getBuildFs) + + async function visitIndex(hash: string) { + const refs = new Set() + const { index } = await getFsIndex(hash) + for (const [name, file] of Object.entries(index.files)) { + const storeHash = file.storeHash ?? index.stores[file.store].hash + const key = `${storeHash}:${file.hash}` + refs.add(`object:${key}` as Ref) + refs.add(`store:${storeHash}` as Ref) + } + + for (const [key, store] of Object.entries(index.stores)) { + refs.add(`store:${store.hash}` as Ref) + } + + if (index.dependencies) { + for (const [k, v] of Object.entries(index.dependencies)) { + refs.add(`index:${v}` as Ref) + } + } + + return refs + } + + function getCommitRefs(commit: Head, ttl?: number) { + const refs = new Set() + refs.add(`index:${commit.storeHash}` as Ref) + + if (commit.programHash) { + refs.add(`index:${commit.programHash}` as Ref) + } + + if (commit.previousCommit) { + const prevRef = `commit:${commit.previousCommit}` as Ref + refs.add(prevRef) + + if (ttl) { + commitTtl.set(commit.previousCommit, ttl) + } + } + + return refs + } + + async function visitCommit(hash: string) { + const ttl = commitTtl.get(hash) + if (ttl === 1) { + return new Set() + } + + const commit = await readJsonRaw(repo, hash) + if (ttl === undefined) { + return getCommitRefs(commit) + } + + return getCommitRefs(commit, ttl - 1) + } + + async function visitHead(id: string) { + const h = await repo.getHead(id) + if (!h) { + throw new Error(`No head found: ${id}`) + } + + return getCommitRefs(h, maxCommits) + } + + async function _getDependencies(parsed: ReturnType) { + switch (parsed.type) { + case 'head': + return visitHead(parsed.id) + case 'commit': + return visitCommit(parsed.hash) + case 'index': + return visitIndex(parsed.hash) + case 'store': + return visitStore(parsed.hash) + case 'object': + return visitObject(parsed.hash, parsed.store) + default: + throw new Error(`Not implemented: ${parsed.type}`) + } + } + + async function getDependencies(ref: Ref) { + if (fromRefs.has(ref) && visited.has(ref)) { + return fromRefs.get(ref)! + } + + visited.add(ref) + const parsed = parseRef(ref) + const info = await getRefInfo(parsed) + if (info.status !== 'ok') { + throw new Error(`Ref is in an invalid state: ${info.status}`) + } + + const refs = await _getDependencies(parsed) + + for (const to of refs) { + addRef(ref, to) + } + + return refs + } + + // function getDependents(ref: Ref) { + // const refs = toRefs.get(ref) + // if (!refs) { + // throw new Error(`Ref not found: ${ref}`) + // } + + // return refs + // } + + async function resolveRef(ref: Ref) { + const parsed = parseRef(ref) + const info = await getRefInfo(parsed) + + return { + type: parsed.type, + size: info.ownSize, + status: info.status, + hash: parsed.hash, + } + } + + return { getDependencies, resolveRef } +} + +async function visitHead(walker: RepoWalker, id: string) { + const stats = initStats() + const refs = await getAllRefs(walker, `head:${id}`) + for (const [k, r] of refs) { + if (!r.hash) continue + + if (r.status !== 'ok') { + switch (r.status) { + case 'missing': + stats.missing?.add(r.hash) + break + case 'corrupted': + stats.corrupted?.add(r.hash) + break + + default: + getLogger().log(`Found valid object with invalid store: ${r.hash} [status: ${r.status}]`) + } + + continue + } + + if (r.type === 'commit') { + stats.commits.add(r.hash) + } + if (r.type === 'store') { + stats.stores.add(r.hash) + } + if (r.type === 'index') { + stats.indices.add(r.hash) + } + if (r.type === 'object') { + stats.objects.add(r.hash) + stats.stores.add(parseRef(k).store!) + } + } + + return stats +} + +// interface CachedStats { +// readonly commitHash: string +// readonly stats: BuildFsStats +// } + +// async function getCachedStats(repo: DataRepository, id: string): Promise { +// const path = require('node:path') +// const cacheDir = path.resolve(repo.getDataDir(), '..', '_stats_cache') +// const cached = await getFs().readFile(path.resolve(cacheDir, id), 'utf-8').catch(throwIfNotFileNotFoundError) +// if (!cached) { +// return +// } + +// const d = JSON.parse(cached) + +// return { +// commitHash: d.commitHash, +// stats: Object.fromEntries(Object.entries(d.stats).map(([k, v]) => [k, new Set(v as any)])) as any, +// } +// } + +// async function setCachedStats(repo: DataRepository, id: string, data: CachedStats): Promise { +// const path = require('node:path') +// const cacheDir = path.resolve(repo.getDataDir(), '..', '_stats_cache') +// await getFs().writeFile(path.resolve(cacheDir, id), JSON.stringify({ +// commitHash: data.commitHash, +// stats: Object.fromEntries(Object.entries(data.stats).map(([k, v]) => [k, Array.from(v)])) as any +// })) +// } + +export async function collectStats(repo: DataRepository, maxCommits?: number) { + const results: RepoStats = {} + const walker = createRepoWalker(repo, maxCommits) + + + for (const h of await repo.listHeads()) { + if (!(h.id in results)) { + const r = await visitHead(walker, h.id) + results[h.id] = { ...r, type: 'unknown' } + } + } + + return results +} + +function decode(buf: Uint8Array, encoding: BufferEncoding = 'utf-8') { + if (Buffer.isBuffer(buf)) { + return buf.toString(encoding) + } + + return Buffer.from(buf).toString(encoding) +} + +function renderKey(key: PropertyKey) { + if (typeof key === 'string') { + return `${key}:` + } else if (typeof key === 'number') { + return `<${key}>` + } + + if (key.description) { + return `[@@${key.description}]:` + } + + return `[unique symbol]:` +} + +const renderHashWidth = 12 +function renderHash(h: string) { + return h.slice(0, renderHashWidth) +} + +function showJsonDiff(diff: JsonDiff, depth = 0, key?: PropertyKey): void { + const printWithIndent = (s: string, showKey = true, d = depth) => { + const withKey = key && showKey ? `${renderKey(key)} ${s}` : s + printLine(`${' '.repeat(d)}${withKey}`) + } + + switch (diff.kind) { + case 'none': + case 'reference': + if (depth === 0) { + printLine('No differences') + } else if (diff.kind === 'reference') { + // printWithIndent('[ref]') + } else { + printWithIndent('[none]') + } + return + + case 'type': + printWithIndent(`[type] ${colorize('red', diff.left)} ${colorize('green', diff.right)}`) + return + + case 'value': + switch (diff.type) { + case 'string': + printWithIndent(`${colorize('red', diff.left)} ${colorize('green', diff.right)}`) + return + + case 'pointer': + if (diff.metadata.kind !== 'none') { + const metadataDiff = `[metadata: ${colorize('red', renderHash(diff.metadata.left))} ${colorize('green', renderHash(diff.metadata.right))}]` + if (diff.left.hash === diff.right.hash) { + printWithIndent(metadataDiff) + } else { + printWithIndent(`${metadataDiff} ${colorize('red', renderHash(diff.left.hash))} ${colorize('green', renderHash(diff.right.hash))}`) + } + } else { + printWithIndent(`${colorize('red', renderHash(diff.left.hash))} ${colorize('green', renderHash(diff.right.hash))}`) + } + return + + case 'symbol': + case 'bigint': + case 'number': + case 'boolean': + case 'function': + printWithIndent(`${colorize('red', String(diff.left))} ${colorize('green', String(diff.right))}`) + return + + case 'array': + printWithIndent('[') + for (let i = 0; i < diff.diff.length; i++) { + const d = diff.diff[i] + if (d.kind === 'none') continue + + if (d.kind === 'type') { + if (d.left === 'undefined' && diff.length.kind === 'value' && diff.length.left < diff.length.right) { + printWithIndent(renderInsert(`<${i}> ${d.right}`), false, depth + 1) + } else if (d.right === 'undefined' && diff.length.kind === 'value' && diff.length.right < diff.length.left) { + printWithIndent(renderRemove(`<${i}> ${d.left}`), false, depth + 1) + } else { + showJsonDiff(d, depth + 1, i) + } + } else { + showJsonDiff(d, depth + 1, i) + } + } + printWithIndent(']', false) + return + + case 'object': { + printWithIndent('{') + let inserts = 0 + let deletions = 0 + for (const d of diff.diff) { + if (d[0].kind === 'arrangment') { + if (d[0].order.left === -1) { // added + if (d[0].value.kind !== 'none') { + throw new Error(`Key diff not implemented: ${d[0].value.kind}`) + } + inserts += 1 + printWithIndent(renderInsert(`<${d[0].order.right}> ${d[0].value.value}:`), false, depth + 1) + } else if (d[0].order.right === -1) { // removed + if (d[0].value.kind !== 'none') { + throw new Error(`Key diff not implemented: ${d[0].value.kind}`) + } + deletions += 1 + printWithIndent(renderRemove(`<${d[0].order.left}> ${d[0].value.value}:`), false, depth + 1) + } else { + if (d[0].value.kind !== 'none') { + throw new Error(`Key diff not implemented: ${d[0].value.kind}`) + } + + // Don't show the order diff if it's accounted for by inserts/deletions + if (d[0].order.left + inserts === d[0].order.right - deletions) { + showJsonDiff(d[1], depth + 1, `${d[0].value.value}`) + } else { + const orderDiff = `<${colorize('red', String(d[0].order.left))} -> ${colorize('green', String(d[0].order.right))}>` + if (d[1].kind === 'none' || d[1].kind === 'reference') { + printWithIndent(`${orderDiff} ${d[0].value.value}: [same value]`, false, depth + 1) + } else { + showJsonDiff(d[1], depth + 1, `${orderDiff} ${d[0].value.value}`) + } + } + } + + continue + } + + if (d[0].kind !== 'none') { + throw new Error(`Key diff not implemented: ${d[0].kind}`) + } + + if (d[1].kind === 'none') continue + + showJsonDiff(d[1], depth + 1, d[0].value) + } + printWithIndent('}', false) + } + } + } +} + +function diffJson(a: any, b: any) { + const diff = jsonDiff(a, b) + showJsonDiff(diff) +} + +function diffLines(a: string, b: string) { + const l1 = a.split('\n') + const l2 = b.split('\n') + + const r = arrayEditDistance(l1, l2, { + insert: a => a.length, + remove: b => b.length, + update: (a, b) => a.length + b.length, //(a, b) => levenshteinDistance(a, b), + }) + + for (const op of r.ops) { + switch (op[0]) { + case 'insert': + printLine(renderInsert(op[1])) + break + case 'remove': + printLine(renderRemove(op[1])) + break + case 'update': + const r2 = arrayEditDistance(op[1].split(''), op[2].split(''), { + update: (a, b) => 2, + }) + + for (const op2 of r2.ops) { + switch (op2[0]) { + case 'insert': + print(colorize('green', op2[1])) + case 'remove': + print(colorize('red', op2[1])) + break + case 'noop': + print(op2[1]) + break + case 'update': + throw new Error(`Shouldn't happen`) + } + } + + printLine() + + break + } + } +} + +type PrimitiveType = 'object' | 'symbol' | 'number' | 'string' | 'undefined' | 'bigint' | 'function' | 'boolean' +type ExtendedType = PrimitiveType | 'array' | 'null' | 'pointer' + +interface JsonNoDiff { + readonly kind: 'none' + readonly value: any +} + +interface JsonReferenceDiff { + readonly kind: 'reference' + readonly type: 'object' | 'array' // or function + readonly left: any + readonly right: any +} + +interface JsonTypeDiff { + readonly kind: 'type' + readonly left: ExtendedType + readonly right: ExtendedType + readonly leftValue: any + readonly rightValue: any +} + +interface JsonPrimitiveValueDiff { + readonly kind: 'value' + readonly type: Exclude + readonly left: T + readonly right: T +} + +interface JsonStructuredValueDiff { + readonly kind: 'value' + readonly diff: T +} + +interface JsonArrangementDiff { + readonly kind: 'arrangment' + readonly order: JsonNumberDiff + readonly value: Exclude +} + +type JsonKeyDiff = JsonNoDiff | JsonStringDiff | JsonNumberDiff | JsonSymbolDiff | JsonTypeDiff | JsonArrangementDiff + +interface JsonObjectDiff extends JsonStructuredValueDiff<[key: JsonKeyDiff, value: JsonDiff][]> { + readonly type: 'object' +} + +interface JsonArrayDiff extends JsonStructuredValueDiff { + readonly type: 'array' + readonly length: JsonNoDiff | JsonNumberDiff +} + +interface JsonStringDiff extends JsonPrimitiveValueDiff { + readonly type: 'string' +} + +interface JsonNumberDiff extends JsonPrimitiveValueDiff { + readonly type: 'number' +} + +interface JsonBigintDiff extends JsonPrimitiveValueDiff { + readonly type: 'bigint' +} + +interface JsonBooleanDiff extends JsonPrimitiveValueDiff { + readonly type: 'boolean' +} + +interface JsonSymbolDiff extends JsonPrimitiveValueDiff { + readonly type: 'symbol' +} + +interface JsonFunctionDiff extends JsonPrimitiveValueDiff { + readonly type: 'function' +} + +interface JsonPointerDiff extends JsonPrimitiveValueDiff { + readonly type: 'pointer' + readonly metadata: JsonNoDiff | JsonStringDiff +} + +type JsonDiff = + | JsonNoDiff + | JsonReferenceDiff + | JsonPointerDiff + | JsonTypeDiff + | JsonObjectDiff + | JsonArrayDiff + | JsonStringDiff + | JsonNumberDiff + | JsonBigintDiff + | JsonBooleanDiff + | JsonSymbolDiff + | JsonFunctionDiff + +function getExtendedType(o: any): ExtendedType { + if (o === null) { + return 'null' + } else if (Array.isArray(o)) { + return 'array' + } else if (isDataPointer(o)) { + return 'pointer' + } + + return typeof o +} + +function jsonObjectDiff(a: any, b: any): JsonObjectDiff | JsonReferenceDiff { + const diff: [key: JsonKeyDiff, value: JsonDiff][] = [] + const keysA = Object.keys(a) + const keysB = Object.keys(b) + const orderA = Object.fromEntries(keysA.map((k, i) => [k, i])) + const orderB = Object.fromEntries(keysB.map((k, i) => [k, i])) + // A new key in A that appears last should not come before a new key in B that appears first + // Not sure if this works as intended + const getSortOrder = (k: string) => orderB[k] ?? orderA[k] + const keys = new Set([...keysA, ...keysB].sort((a, b) => getSortOrder(a) - getSortOrder(b))) + for (const k of keys) { + const x = orderA[k] ?? -1 + const y = orderB[k] ?? -1 + const orderDiff = jsonDiff(x, y) + const kDiff: JsonKeyDiff = orderDiff.kind !== 'none' + ? { kind: 'arrangment', order: orderDiff as JsonNumberDiff, value: { kind: 'none', value: k } } + : { kind: 'none', value: k} + + diff.push([kDiff, jsonDiff(a[k], b[k])]) + } + + if (diff.every(d => d[0].kind === 'none' && (d[1].kind === 'none' || d[1].kind === 'reference'))) { + return { kind: 'reference', type: 'object', left: a, right: b } + } + + return { + kind: 'value', + type: 'object', + diff, + } +} + +function jsonArrayDiff(a: any[], b: any[]): JsonArrayDiff | JsonReferenceDiff { + const maxLength = Math.max(a.length, b.length) + const diff: JsonDiff[] = Array(maxLength) + const length = jsonDiff(a.length, b.length) as JsonArrayDiff['length'] + for (let i = 0; i < maxLength; i++) { + diff[i] = jsonDiff(a[i], b[i]) + } + + if (diff.every(d => d.kind === 'none' || d.kind === 'reference') && length.kind === 'none') { + return { kind: 'reference', type: 'array', left: a, right: b } + } + + return { + kind: 'value', + type: 'array', + diff, + length, + } +} + +function jsonPointerDiff(a: DataPointer, b: DataPointer): JsonPointerDiff | JsonNoDiff { + if (a.isResolved() && b.isResolved()) { + const l = a.resolve() + const r = b.resolve() + if (l.hash === r.hash) { + if (l.storeHash !== r.storeHash) { + return { + kind: 'value', + type: 'pointer', + left: a, + right: b, + metadata: jsonDiff(l.storeHash, r.storeHash) as JsonStringDiff, + } + } + return { kind: 'none', value: l } + } + } + + return { + kind: 'value', + type: 'pointer', + left: a, + right: b, + metadata: { kind: 'none', value: '' }, // wrong + } +} + + +function jsonDiff(a: any, b: any): JsonDiff { + if (a === b) { + return { kind: 'none', value: a } + } + + // We only do this because reading "raw" objects won't deserialize pointers + a = maybeConvertToPointer(a) + b = maybeConvertToPointer(b) + + const left = getExtendedType(a) + const right = getExtendedType(b) + + if (left !== right) { + return { + kind: 'type', + left, + right, + leftValue: a, + rightValue: b, + } + } + + switch (left) { + case 'symbol': + case 'bigint': + case 'number': + case 'string': + case 'boolean': + case 'function': + return { kind: 'value', type: left, left: a, right: b } + + case 'pointer': + return jsonPointerDiff(a, b) + + case 'array': + return jsonArrayDiff(a, b) + + case 'object': + return jsonObjectDiff(a, b) + + case 'null': + case 'undefined': + throw new Error(`Primitive types possibly from different realms: ${a} !== ${b}`) + } +} + +async function _diffObjects(repo: DataRepository, a: Ref, b: Ref) { + const refA = parseRef(a) + const refB = parseRef(b) + if (refA.type !== 'object') { + throw new Error(`Not an object ref: ${a}`) + } + if (refB.type !== 'object') { + throw new Error(`Not an object ref: ${b}`) + } + + if (refA.hash === refB.hash) { + // TODO: diff stores + return + } + + const datum = await Promise.all([repo.readData(refA.hash), repo.readData(refB.hash)]) + const strA = decode(datum[0]) + const strB = decode(datum[1]) + + // Try diff json + try { + return diffJson(JSON.parse(strA), JSON.parse(strB)) + } catch {} + + return diffLines(strA, strB) +} + +export async function diffObjects(repo: DataRepository, a: string, b: string) { + const resolved = await Promise.all([getArtifactByPrefix(repo, a), getArtifactByPrefix(repo, b)]) + const datum = await Promise.all([repo.readData(resolved[0]), repo.readData(resolved[1])]) + const strA = decode(datum[0]) + const strB = decode(datum[1]) + if (!strA.startsWith('{')) { + return diffLines(strA, strB) + } + + return diffJson(JSON.parse(strA), JSON.parse(strB)) +} + +function renderInsert(s: string) { + return colorize('green', `+ ${s}`) +} + +function renderRemove(s: string) { + return colorize('red', `- ${s}`) +} + +// It's always a === before, b === after +export async function diffIndices(repo: DataRepository, a: string, b: string) { + const walker = createRepoWalker(repo) + const resolved = await Promise.all([getArtifactByPrefix(repo, a), getArtifactByPrefix(repo, b)]) + // const deps = await Promise.all([ + // getAllRefs(walker, `index:${resolved[0]}`), + // getAllRefs(walker, `index:${resolved[1]}`) + // ]) + + // const s1 = new Set(deps[0].keys()) + // const s2 = new Set(deps[1].keys()) + const deps = await Promise.all([ + walker.getDependencies(`index:${resolved[0]}`), + walker.getDependencies(`index:${resolved[1]}`), + ]) + + const s1 = deps[0] + const s2 = deps[1] + + const d1 = diffSets(s1, s2) + const d2 = diffSets(s2, s1) + if (d1.size === 0 && d2.size === 0) { + if (resolved[0] !== resolved[1]) { + printLine('Data is the same but stored differently') + } else { + printLine('No differences') + } + return + } + + for (const r of d1) { + printLine(renderInsert(r)) + } + + for (const r of d2) { + printLine(renderRemove(r)) + } +} + +export async function diffFileInCommit(repo: DataRepository, fileName: string, commit: Head, previousCommit?: Head) { + const currentIndex = await repo.getBuildFs(commit.storeHash) + previousCommit ??= commit.previousCommit + ? JSON.parse(Buffer.from(await repo.readData(commit.previousCommit)).toString('utf-8')) + : undefined + + const previousIndex = previousCommit ? await repo.getBuildFs(previousCommit.storeHash) : undefined + const currentFileHash = currentIndex.index.files[fileName]?.hash + const previousFileHash = previousIndex?.index.files[fileName]?.hash + + const currentFile = currentFileHash ? JSON.parse(Buffer.from(await repo.readData(currentFileHash)).toString('utf-8')) : undefined + const previousFile = previousFileHash ? JSON.parse(Buffer.from(await repo.readData(previousFileHash)).toString('utf-8')) : undefined + if (currentFile === previousFile) { + printLine(colorize('brightRed', 'File does not exist')) + } else { + diffJson(previousFile, currentFile) + } +} + +export async function diffFileInLatestCommit(fileName: string, opt?: { commitsBack?: number }) { + const repo = getDataRepository() + const commits = await listCommits(getBuildTargetOrThrow().programId) + const latestCommit = commits[0] + if (!latestCommit) { + printLine(colorize('brightRed', 'No commit found')) + return + } + + if (opt?.commitsBack) { + const previousCommit = commits[opt.commitsBack] + if (!previousCommit) { + throw new Error(`No commit found: ${opt.commitsBack}`) + } + + return await diffFileInCommit(repo, fileName, latestCommit, previousCommit) + } + + + await diffFileInCommit(repo, fileName, latestCommit) +} + +const statData = (hash: string) => getDataRepository().statData(hash) + +function printKB(byteCount: number) { + return `${Math.floor(byteCount / 1024)} kB` +} + +async function getTotalSize(hashes: Iterable) { + const stats = await Promise.all([...hashes].map(statData)) + + return stats.reduce((a, b) => a + b.size, 0) +} + +export async function printStats(k: string, v: HeadStats) { + const objectsSize = await getTotalSize(v.objects) + const storesSize = await getTotalSize(v.stores) + const indicesSize = await getTotalSize(v.indices) + const commitsSize = await getTotalSize(v.commits) + + printLine(`ID: ${k} [type: ${v.type}]`) + printLine(' # of objects', v.objects.size, `(${printKB(objectsSize)})`) + printLine(' # of stores', v.stores.size, `(${printKB(storesSize)})`) + printLine(' # of indices', v.indices.size, `(${printKB(indicesSize)})`) + printLine(' # of commits', v.commits.size, `(${printKB(commitsSize)})`) + + return { + sizes: { + objects: objectsSize, + stores: storesSize, + indices: indicesSize, + commits: commitsSize, + } + } +} + + + +interface GcEvent extends BaseOutputMessage { + readonly type: 'gc' + readonly discovered?: number + readonly deleted?: number +} + + +interface GcSummaryEvent extends BaseOutputMessage { + readonly type: 'gc-summary' + readonly numObjects: number + readonly numDirs: number + readonly dryrun?: boolean +} + +export function getEventLogger() { + const emitter = getTypedEventEmitter('gc') + const emitter2 = getTypedEventEmitter('gc-summary') + + return { + onGc: emitter.on, + onGcSummary: emitter2.on, + emitGcEvent: emitter.fire, + emitGcSummaryEvent: emitter2.fire, + } +} diff --git a/src/build-fs/utils.ts b/src/build-fs/utils.ts new file mode 100644 index 0000000..87711e4 --- /dev/null +++ b/src/build-fs/utils.ts @@ -0,0 +1,65 @@ +import * as path from 'node:path' +import { getFs } from '../execution' +import { DataRepository, getDataRepository, getPrefixedPath } from '../artifacts' + +export async function findArtifactByPrefix(repo: DataRepository, prefix: string) { + if (prefix.length === 64) { + return prefix + } + + const prefixedPath = getPrefixedPath(prefix) + const dataDir = repo.getDataDir() + const dirPath = path.resolve(dataDir, path.dirname(prefixedPath)) + const basename = path.basename(prefixedPath) + const matches: string[] = [] + for (const f of await getFs().readDirectory(dirPath)) { + if (f.type === 'file' && f.name.startsWith(basename)) { + matches.push(f.name) + } + } + + if (matches.length > 1) { + throw new Error(`Ambiguous match: ${matches.join('\n')}`) + } + + if (!matches[0]) { + return + } + + const rem = path.relative(dataDir, dirPath).split('/').join('') + + return rem + matches[0] +} + +export async function getArtifactByPrefix(repo: DataRepository, prefix: string) { + const r = await findArtifactByPrefix(repo, prefix) + if (!r) { + throw new Error(`Object not found: ${prefix}`) + } + + return r +} + + +export function getObjectByPrefix(prefix: string, repo = getDataRepository()) { + return getArtifactByPrefix(repo, prefix.replace(/\//g, '')) +} + + +export async function getMetadata(repo: DataRepository, target: string) { + const [source, hash] = target.split(':') + + const resolvedSource = await findArtifactByPrefix(repo, source) + const resolvedHash = await findArtifactByPrefix(repo, hash) + + if (!resolvedSource) { + throw new Error(`Did not find source matching hash: ${source}`) + } + + if (!resolvedHash) { + throw new Error(`Did not find object matching hash: ${hash}`) + } + + return repo.getMetadata(resolvedHash, resolvedSource) +} + diff --git a/src/build/builder.ts b/src/build/builder.ts new file mode 100644 index 0000000..dd1f0c4 --- /dev/null +++ b/src/build/builder.ts @@ -0,0 +1,99 @@ + +import * as os from 'node:os' + +// Terminology: +// * Project - source code location +// * Package - bundled distributable artifacts +// * Build - a graph node that consumes inputs to produce artifacts +// * Defines - build-time variables + +// Dependencies are always represented as artifacts rather than the producer of said artifacts +// * In some cases, the package is the artifact (e.g. `npm` packages) + +export interface BuildTarget { + mode?: 'debug' | 'release' + os?: string + arch?: string + runtime?: string // Qualifies shared libs e.g. `glibc`, `node`, `browser` are all runtimes +} + + +export type Os = 'linux' | 'darwin' | 'windows' | 'freebsd' +export type Arch = 'aarch64' | 'x64' + +export interface QualifiedBuildTarget { + readonly os: Os + readonly arch: Arch + readonly endianness: 'LE' | 'BE' + readonly libc?: 'musl' +} + +function parseOs(osType: string): Os { + switch (osType) { + case 'Darwin': + return 'darwin' + case 'Linux': + return 'linux' + case 'Windows_NT': + return 'windows' + + default: + throw new Error(`OS not supported: ${osType}`) + } +} + +function parseArch(arch: string): Arch { + switch (arch) { + case 'arm64': + return 'aarch64' + case 'x64': + return arch + + default: + throw new Error(`Architecture not supported: ${arch}`) + } +} + +export function resolveBuildTarget(target?: Partial): QualifiedBuildTarget { + const _os = target?.os ?? parseOs(os.type()) + const arch = target?.arch ?? parseArch(os.arch()) + const endianness = target?.endianness ?? os.endianness() + + return { + ...target, + os: _os, + arch, + endianness, + } +} + +export function toNodeArch(arch: Arch): NodeJS.Architecture { + switch (arch) { + case 'aarch64': + return 'arm64' + + default: + return arch + } +} + +export function toNodePlatform(os: Os): NodeJS.Platform { + switch (os) { + case 'windows': + return 'win32' + + default: + return os + } +} + +export interface CommonParams { + readonly defines?: Record + readonly target?: BuildTarget +} + +export interface BuildSourceParams extends CommonParams { + readonly sourceDir: string + readonly output?: string +} + diff --git a/src/build/go.ts b/src/build/go.ts new file mode 100644 index 0000000..47f676f --- /dev/null +++ b/src/build/go.ts @@ -0,0 +1,151 @@ +import { runCommand } from "../utils/process" +import { BuildSourceParams } from "./builder" + +interface Overlay { + Replace: Record +} + +// -C dir <-- changes to this dir, must be first flag +// -race +// -msan +// -asan +// -installsuffix suffix + +// go tool dist list + +type Os = + | 'linux' + | 'darwin' + | 'windows' + | 'solaris' + | 'plan9' + | 'openbsd' + | 'netbsd' + | 'freebsd' + | 'ios' // mobile + | 'android' // mobile + + | 'aix' + | 'illumos' + | 'dragonfly' + + | 'js' // wasm + +type Arch = + | 'arm' + | '386' + | 'arm64' + | 'amd64' + +type BuildMode = + | 'archive' + | 'c-archive' + | 'c-shared' + | 'default' + | 'shared' + | 'exe' + | 'pie' + | 'plugin' + +// CGO_ENABLED=0 + +interface RawBuildParams { + // -overlay file + // -pgo file + // -gccgoflags + // -gcflags + // -ldflags + + cwd?: string + cgo?: boolean + os?: string + arch?: string + output?: string // File or directory + moduleMode?: 'readonly' | 'vendor' | 'mod' + ldflags?: string // https://pkg.go.dev/cmd/link + trimpath?: boolean + // asmflags?: string + // compiler?: 'gc' | 'gccgo' + // packages?: string[] +} + +async function runGoBuild(params: RawBuildParams) { + const env: Record = { ...process.env } + if (params.cgo === false) { + env['CGO_ENABLED'] = '0' + } + + if (params.os) { + env['GOOS'] = params.os + } + + if (params.arch) { + env['GOARCH'] = params.arch + } + + const args = ['build'] + if (params.trimpath) { + args.push('-trimpath') + } + + if (params.moduleMode) { + args.push(`-mod`, params.moduleMode) + } + + if (params.ldflags) { + args.push('-ldflags', `"${params.ldflags}"`) + } + + if (params.output) { + args.push('-o', params.output) + } + + await runCommand('go', args, { env, cwd: params.cwd }) +} + + +function resolveParams(params: BuildSourceParams): RawBuildParams { + const res: RawBuildParams = { + cwd: params.sourceDir, + moduleMode: 'readonly', + } + + const ldflags: string[] = [] + + if (params.target) { + if (params.target.mode === 'release') { + ldflags.push('-s', '-w') + res.trimpath = true + res.cgo = false + } + + res.os = params.target.os + res.arch = params.target.arch + if (res.arch === 'aarch64') { + res.arch = 'arm64' + } else if (res.arch === 'x64') { + res.arch = 'amd64' + } + } + + if (params.defines) { + for (const [k, v] of Object.entries(params.defines)) { + ldflags.push('-X', `'${k}=${v}'`) + } + } + + if (ldflags.length > 0) { + res.ldflags = ldflags.join(' ') + } + + if (params.output) { + res.output = params.output + } + + return res +} + +export async function buildGoProgram(params: BuildSourceParams) { + const resolved = resolveParams(params) + await runGoBuild(resolved) +} \ No newline at end of file diff --git a/src/build/sea.ts b/src/build/sea.ts new file mode 100644 index 0000000..7b8fe4f --- /dev/null +++ b/src/build/sea.ts @@ -0,0 +1,139 @@ +import * as path from 'node:path' +import { getFs } from '../execution' +import { runCommand } from '../utils/process' +import { getHash, makeExecutable, throwIfNotFileNotFoundError } from '../utils' +import { createRequire } from 'node:module' +import { getLogger } from '..' +import { seaAssetPrefix } from '../bundler' +import { isDataPointer } from '../build-fs/pointers' +import { getDataRepository } from '../artifacts' + +interface PostjectOptions { + readonly overwrite?: boolean + readonly sentinelFuse?: string + readonly machoSegmentName?: string +} + +interface Postject { + inject: (filename: string, resourceName: string, resourceData: Buffer, options?: PostjectOptions) => Promise + remove?: (filename: string, resourceName: string, options?: PostjectOptions) => Promise +} + +function getPostject(): Postject { + // This is so we can still load `postject` as an SEA + const loadFromCwd = () => createRequire(path.resolve(process.cwd(), 'package.json'))('postject') + + try { + return require('postject') + } catch { + return loadFromCwd() + } +} + +// This is split so it doesn't get treated as the sentinel +const sentinelFuseParts = ['NODE_SEA_FUSE', 'fce680ab2cc467b6e072b8b5df1996b2'] + +async function injectSeaBlob(executable: string, blob: Buffer) { + await getPostject().inject(executable, 'NODE_SEA_BLOB', blob, { + overwrite: true, + machoSegmentName: 'NODE_SEA', // Darwin only + sentinelFuse: sentinelFuseParts.join('_'), + }) +} + +async function removeSeaBlob(executable: string) { + const postject = getPostject() + if (!postject.remove) { + getLogger().log('Missing `remove` method from `postject`') + return + } + + const didRemove = await postject.remove(executable, 'NODE_SEA_BLOB', { + machoSegmentName: 'NODE_SEA', // Darwin only + sentinelFuse: sentinelFuseParts.join('_'), + }) + + if (!didRemove) { + getLogger().log('No existing SEA blob found') + return + } + + if (process.platform === 'darwin') { + await runCommand('codesign', ['--sign', '-', executable]) + } +} + +interface SeaConfig { + readonly main: string + readonly output: string + readonly disableExperimentalSEAWarning?: boolean + readonly useSnapshot?: boolean + readonly useCodeCache?: boolean + readonly assets?: Record +} + +export async function resolveAssets(assets?: Record): Promise | undefined> { + if (!assets) { + return + } + + const resolved: Record = {} + for (const [k, v] of Object.entries(assets)) { + if (!k.startsWith(seaAssetPrefix)) { + throw new Error(`Invalid asset name: ${k}. Expected prefix "${seaAssetPrefix}".`) + } + + const name = k.slice(seaAssetPrefix.length) + if (isDataPointer(v)) { + resolved[name] = await getDataRepository().resolveArtifact(v.hash) + } else { + resolved[name] = path.resolve(v) + } + } + + return resolved +} + +export async function makeSea(main: string, nodePath: string, dest: string, assets?: Record, sign = true) { + const output = path.resolve(path.dirname(dest), 'sea-prep.blob') + const config: SeaConfig = { + main, + output, + assets: await resolveAssets(assets), + useSnapshot: true, + disableExperimentalSEAWarning: true, + } + + getLogger().log(`SEA assets`, config.assets) + + const configPath = path.resolve(path.dirname(dest), 'sea-config.json') + await getFs().writeFile(configPath, JSON.stringify(config)) + + try { + await getFs().writeFile(dest, await getFs().readFile(nodePath)) + await makeExecutable(dest) + await removeSeaBlob(dest) + + getLogger().log('Creating SEA blob using executable', nodePath) + await runCommand(dest, ['--experimental-sea-config', configPath], { + cwd: path.dirname(main), + env: { ...process.env, BUILDING_SEA: '1' }, + shell: false, + // stdio: 'inherit', + }) + + getLogger().log('Injecting SEA blob') + const blob = await getFs().readFile(output) + await injectSeaBlob(dest, Buffer.from(blob)) + if (process.platform === 'darwin' && sign) { + await runCommand('codesign', ['--sign', '-', dest]) + } + } finally { + await Promise.all([ + getFs().deleteFile(configPath).catch(throwIfNotFileNotFoundError), + getFs().deleteFile(output).catch(throwIfNotFileNotFoundError) + ]) + } +} + + diff --git a/src/build/sources.ts b/src/build/sources.ts new file mode 100644 index 0000000..c8d82d4 --- /dev/null +++ b/src/build/sources.ts @@ -0,0 +1,31 @@ +import * as git from '../git' +import * as path from 'node:path' +import { getUserSynapseDirectory } from '../workspaces' +import { getFs } from '../execution' +import { ensureDir, getHash } from '../utils' +import { runCommand } from '../utils/process' +import { getLogger } from '..' + +const getSourcesDirs = () => path.resolve(getUserSynapseDirectory(), 'build', 'sources') +const getPkgName = (url: string) => url.replace(/^https?:\/\//, '').replace(/\.git$/, '') + +interface GitSource { + readonly type: 'git' + readonly url: string + readonly commitish: string +} + +export async function downloadSource(source: GitSource) { + const dest = path.resolve(getSourcesDirs(), getPkgName(source.url), source.commitish) + const fs = getFs() + if (await fs.fileExists(dest)) { + await git.fetchOriginHead(dest, source.commitish) + + return dest + } + + await git.fetchRepo(dest, source.url, source.commitish) + + return dest +} + diff --git a/src/bundler.ts b/src/bundler.ts new file mode 100644 index 0000000..e7a86f8 --- /dev/null +++ b/src/bundler.ts @@ -0,0 +1,1713 @@ +import ts from 'typescript' +import * as path from 'node:path' +import * as esbuild from 'esbuild' +import { Fs, SyncFs } from './system' +import type { ReflectionOperation, ExternalValue } from './runtime/modules/serdes' +import { createArrayLiteral, createLiteral, createObjectLiteral, createSymbolPropertyName, createVariableStatement, memoize, printNodes, throwIfNotFileNotFoundError } from './utils' +import { topoSort } from './static-solver/scopes' +import { getLogger } from './logging' +import { Artifact, getDataRepository, getDeploymentFs } from './artifacts' +import { PackageService, pruneManifest } from './pm/packages' +import { ModuleResolver } from './runtime/resolver' +import { isBuiltin } from 'node:module' +import { TerraformPackageManifest } from './runtime/modules/terraform' +import { SourceMapV3, toInline } from './runtime/sourceMaps' +import { createModuleResolverForBundling } from './runtime/rootLoader' +import { getWorkingDirectory } from './workspaces' +import { pointerPrefix, createPointer, isDataPointer, toAbsolute, DataPointer, coerceToPointer, isNullHash, applyPointers } from './build-fs/pointers' +import { getModuleType } from './static-solver' +import { readKeySync } from './cli/config' +import { isSelfSea } from './execution' + +// Note: `//!` or `/*!` are considered "legal comments" +// Using "linked" or "external" for `legalComments` creates a `.LEGAL.txt` file + +export interface BundleOptions extends MemCompileOptions { + readonly external?: string[] + readonly bundled?: boolean + readonly minify?: boolean + readonly minifyKeepWhitespace?: boolean + readonly banners?: string[] + readonly legalComments?: 'inline' | 'external' | 'linked' + readonly sourcemap?: boolean | 'inline' | 'external' | 'linked' + readonly sourcesContent?: boolean + readonly splitting?: boolean // Does nothing without multiple entrypoints + + readonly lazyLoad?: string[] + readonly lazyLoad2?: string[] + + // TODO: maybe implement this + // Always lazily-load deployed modules + // This uses a "smart" loader that eagerly loads non-function primitive exports if known ahead of time. + readonly lazyLoadDeployed?: boolean + + readonly executableType?: 'node-script' + readonly serializerHost?: SerializerHost +} + +export interface MemCompileOptions { + readonly compilerOptions?: ts.CompilerOptions + readonly outfile?: string + readonly outdir?: string + readonly moduleTarget?: 'cjs' | 'esm' + readonly platform?: 'browser' | 'node' + readonly allowOverwrite?: boolean + readonly nodePaths?: string + readonly workingDirectory?: string + readonly writeToDisk?: boolean +} + +export function createProgram( + fs: SyncFs, + files: string[], + compilerOptions: ts.CompilerOptions = {} +) { + const host = createCompilerHost(compilerOptions, fs) + + return { + host, + program: ts.createProgram(files, compilerOptions, host), + } +} + +export const setupEsbuild = memoize(() => { + const esbuildPath = readKeySync('esbuild.path') + if (!esbuildPath) { + if (!process.env.ESBUILD_BINARY_PATH && isSelfSea()) { + throw new Error(`Missing esbuild binary`) + } + + getLogger().warn(`No esbuild path found`) + + return + } + + getLogger().debug(`Using esbuild path`, esbuildPath) + + // Updating the env variable doesn't work for `deploy` (??????????) + // Feels like it's a very obscure bug with v8 relating to proxies + // Using a non-enumerable field on `process` works though + Object.defineProperty(process, 'SYNAPSE_ESBUILD_BINARY_PATH', { + value: esbuildPath, + enumerable: false, + }) + + // Triggers a lazy load + + return esbuild.version +}) + +function mapTarget(target: ts.ScriptTarget | undefined) { + if (target === undefined) { + return + } + + switch (target) { + case ts.ScriptTarget.ES2015: + return 'es2015' + case ts.ScriptTarget.ES2016: + return 'es2016' + case ts.ScriptTarget.ES2017: + return 'es2017' + case ts.ScriptTarget.ES2018: + return 'es2018' + case ts.ScriptTarget.ES2019: + return 'es2019' + case ts.ScriptTarget.ES2020: + return 'es2020' + case ts.ScriptTarget.ES2021: + return 'es2021' + case ts.ScriptTarget.ES2022: + return 'es2022' + case ts.ScriptTarget.ESNext: + return 'esnext' + case ts.ScriptTarget.JSON: + return 'json' + case ts.ScriptTarget.Latest: + return 'latest' + case ts.ScriptTarget.ES3: + return 'es3' + case ts.ScriptTarget.ES5: + return 'es5' + default: + throw new Error(`unknown target: ${target}`) + } +} + +function makeBanner(options: BundleOptions) { + const lines: string[] = [] + if (options.banners) { + lines.push(...options.banners) + } + + if (options.compilerOptions?.alwaysStrict && options.moduleTarget !== 'esm') { + lines.push('"use strict";') + } + + // import path from 'node:path' + // import { fileURLToPath } from 'node:url'; + // const __filename = fileURLToPath(import.meta.url) + // const __dirname = path.dirname(__filename) + + // XXX: allows `require` to still work in ESM bundles + if (options.moduleTarget === 'esm' && options.bundled !== false && options.platform !== 'browser') { + lines.push( + ...` +import { createRequire as __createRequire } from 'node:module'; +const require = __createRequire(import.meta.url); +`.trim().split('\n') + ) + } + + if (lines.length === 0) { + return undefined + } + + return { + 'js': lines.join('\n'), + } +} + +interface TranspileOptions { + readonly oldSourcemap?: SourceMapV3 + readonly sourcemapType?: 'external' | 'linked' + readonly workingDirectory?: string + readonly bundleOptions?: BundleOptions +} + +export function createTranspiler(fs: Fs & SyncFs, resolver?: ModuleResolver, compilerOptions?: ts.CompilerOptions) { + const moduleTarget = getModuleType(compilerOptions?.module) + + async function transpile(path: string, data: string | Uint8Array, outfile: string, options?: TranspileOptions) { + const workingDirectory = options?.workingDirectory ?? compilerOptions?.rootDir ?? getWorkingDirectory() + const withSourcemap = options?.oldSourcemap ? `${data}\n\n${toInline(options?.oldSourcemap)}` : data + await fs.writeFile(path, withSourcemap, { fsKey: '#mem' }) + const res = await build( + fs, + resolver ?? createModuleResolverForBundling(fs, workingDirectory), + [path], + { + outfile, + bundled: false, + workingDirectory, + allowOverwrite: true, + compilerOptions, + writeToDisk: false, + sourcemap: options?.sourcemapType, + sourcesContent: false, + moduleTarget, + ...options?.bundleOptions, + } + ) + + // TODO: this is needed for `watch` but it causes flaky synths (?) + // await fs.deleteFile(path, { fsKey: '#mem' }).catch(throwIfNotFileNotFoundError) + + const result = res.outputFiles.find(x => x.path.endsWith(outfile)) + if (result === undefined) { + throw new Error(`Unable to find build artifact: ${outfile}`) + } + + const sourcemap = res.outputFiles.find(x => x.path.endsWith(`${outfile}.map`)) + if (options?.sourcemapType !== undefined && sourcemap === undefined) { + throw new Error(`Unable to find sourcemap: ${outfile}.map`) + } + + return { + result, + sourcemap, + } + } + + return { transpile, moduleTarget } +} + +async function build(fs: Fs & SyncFs, resolver: ModuleResolver, files: string[], options: BundleOptions) { + setupEsbuild() + + const jsx = options.compilerOptions?.jsx + const result = await esbuild.build({ + entryPoints: files, + absWorkingDir: options.workingDirectory, + bundle: options.bundled !== false, + format: options.moduleTarget ?? 'cjs', + platform: options.platform ?? 'node', + target: mapTarget(options.compilerOptions?.target), + outfile: options.outfile, + outdir: options.outdir, + treeShaking: true, + minify: options.minify, + plugins: [createFsPlugin(fs, resolver, options)], + external: options.external, + minifyIdentifiers: options.minifyKeepWhitespace, + minifySyntax: options.minifyKeepWhitespace, + mainFields: (options.moduleTarget === 'esm' && options.platform !== 'browser') + ? ['module', 'main'] + : undefined, + loader: { + '.ttf': 'file', + }, + // Can't be used w/ outfile + // outdir: options.compilerOptions?.outDir, + jsx: (jsx === ts.JsxEmit.ReactJSX || jsx === ts.JsxEmit.ReactJSXDev) ? 'automatic' : undefined, + jsxImportSource: options.compilerOptions?.jsxImportSource, + banner: makeBanner(options), + allowOverwrite: options.allowOverwrite, + inject: [], + write: false, + sourcesContent: options.sourcesContent, + sourcemap: options.sourcemap, + splitting: options.splitting, + // TODO: determine if using `silence` still returns errors/warnings + logLevel: 'silent', + legalComments: options.legalComments, + }) + + if (options.writeToDisk !== false) { + await Promise.all(result.outputFiles.map(f => fs.writeFile(f.path, f.contents))) + } + + for (const warn of result.warnings) { + getLogger().warn(`[esbuild]: ${await esbuild.formatMessages([warn], { kind: 'warning' })}`) + } + + for (const err of result.errors) { + getLogger().error(`[err]: ${await esbuild.formatMessages([err], { kind: 'error' })}`) + } + + if (result.errors.length > 0) { + throw new Error(`Failed to compile file: ${files[0]}`) + } + + return result +} + +export function createPointerMapper() { + const mapping = new Map() + const unmapping = new Map() + function getMappedPointer(p: string) { + if (unmapping.has(p)) { + return p + } + + if (mapping.has(p)) { + return mapping.get(p)! + } + + const id = `${pointerPrefix}${mapping.size}` + mapping.set(p, id) + unmapping.set(id, p) + + return id + } + + return { + getMappedPointer, + getUnmappedPointer: (p: string) => unmapping.get(p) ?? p, + } +} + +export const seaAssetPrefix = `sea-asset:` +export const rawSeaAssetPrefix = `raw-sea-asset:` +export const backendBundlePrefix = `file:./_assets/` + +export type Optimizer = (table: Record, captured: any) => { table: Record, captured: any } + +export function createSerializerHost(fs: { writeDataSync: (data: Uint8Array) => DataPointer }, packagingMode: 'sea' | 'backend-bundle' = 'sea', optimizer?: Optimizer) { + const assets = new Map() + + function getUrl(p: DataPointer) { + switch (packagingMode) { + case 'sea': + return `${seaAssetPrefix}${p.hash}` + case 'backend-bundle': + return `${backendBundlePrefix}${p.hash}` + + default: + throw new Error(`Unknown packaging mode: ${packagingMode}`) + } + } + + function addAsset(p: DataPointer): string { + if (assets.has(p)) { + return assets.get(p)! + } + + const url = getUrl(p) + assets.set(p, url) + + return url + } + + function addRawAsset(data: ArrayBuffer): DataPointer { + const pointer = fs.writeDataSync(new Uint8Array(data)) + addAsset(pointer) + + return pointer + } + + function getAssets() { + return Object.fromEntries( + [...assets.entries()].map(([k, v]) => [v, k] as const) + ) + } + + return { + addAsset, + addRawAsset, + getAssets, + optimize: optimizer, + ...createPointerMapper(), + } +} + +function createFsPlugin(fs: Fs & SyncFs, resolver: ModuleResolver, opt: BundleOptions): esbuild.Plugin { + const serializerHost = opt?.serializerHost + const lazy3Importers = new Map() + + async function resolveJsFile(args: esbuild.OnResolveArgs) { + const resolved = resolver.getFilePath(args.path) + const patchFn = resolver.getPatchFn(resolved) + + if (patchFn) { + return { + path: resolved, + namespace: 'patched', + pluginData: patchFn, + } + } + + if (opt.bundled !== false && args.path.endsWith('.js')) { + const resolvedDeployed = resolved.replace(/\.js$/, '.resolved.js') + if (await fs.fileExists(resolvedDeployed)) { + getLogger().log('Using resolved path for bundling:', resolvedDeployed) + return { + path: resolvedDeployed, + } + } + } + + return { + path: args.path, + // TODO: this is needed for tree-shaking ESM + // sideEffects: false, + } + } + + return { + name: 'mem', + setup(build) { + build.onResolve({ filter: /^pointer:.*/ }, async args => { + const importer = args.pluginData?.virtualId ?? args.importer + const mapped = serializerHost?.getMappedPointer?.(args.path) ?? args.path + + return { + namespace: 'pointer', + path: mapped.slice(pointerPrefix.length), + pluginData: { virtualId: resolver.resolveVirtual(serializerHost?.getUnmappedPointer?.(args.path) ?? args.path, importer) } + } + }) + + build.onResolve({ filter: /^raw-sea-asset:.*/ }, async args => { + const importer = args.pluginData?.virtualId ?? args.importer + + return { + namespace: rawSeaAssetPrefix.slice(0, -1), + path: resolver.resolve(args.path.slice(rawSeaAssetPrefix.length), importer), + } + }) + + build.onResolve({ filter: /^synapse:.*/ }, async args => { + return { + path: resolver.resolve(args.path), + } + }) + + build.onResolve({ filter: /^lazy3:.*/ }, async args => { + const mode: 'esm' | 'cjs' = args.kind === 'import-statement' ? 'esm' + : args.kind === 'require-call' ? 'cjs' + : opt.moduleTarget ?? 'cjs' + + try { + const p = args.path.slice(6) + const importer = lazy3Importers.get(p) + if (!importer) { + throw new Error(`No importer found for path: ${p}`) + } + + const filePath = resolver.resolveVirtual(p, importer, mode) + + if (filePath.match(/\.(js|ts|tsx)$/)) { + return resolveJsFile({ ...args, path: filePath }) + } + + return { path: resolver.resolve(filePath) } + } catch (e) { + getLogger().warn(`Deferring to esbuild for unresolved module "${args.path}"`, e) + } + }) + + function matchPattern(spec: string, pattern: string) { + if (isBuiltin(spec) && !spec.startsWith('node:')) { + spec = `node:${spec}` + } + + if (pattern.endsWith('*')) { + return spec.startsWith(pattern.slice(0, -1)) + } + + return spec.startsWith(pattern) + } + + build.onResolve({ filter: /.*/ }, async (args) => { + if (opt.bundled === false) { + return { path: args.path } + } + + if (opt.lazyLoad?.some(x => matchPattern(args.path, x))) { + return { namespace: 'lazy', path: args.path } + } + + if (opt.lazyLoad2?.some(x => matchPattern(args.path, x))) { + lazy3Importers.set(args.path, args.importer) + return { namespace: 'lazy2', path: args.path } + } + + if (isBuiltin(args.path)) { + return + } + + if (opt.external?.some(x => args.path.startsWith(x))) { + return + } + + const mode: 'esm' | 'cjs' = args.kind === 'import-statement' ? 'esm' + : args.kind === 'require-call' ? 'cjs' + : opt.moduleTarget ?? 'cjs' + + const importer = args.pluginData?.virtualId ?? args.importer + try { + const filePath = resolver.resolveVirtual(args.path, importer, mode) + if (filePath.match(/\.(js|ts|tsx)$/)) { + return resolveJsFile({ ...args, path: filePath }) + } + + return { path: resolver.resolve(filePath) } + } catch (e) { + getLogger().warn(`Deferring to esbuild for unresolved module "${args.path}" `, e) + } + }) + + build.onLoad({ namespace: 'lazy', filter: /.*/ }, async args => { + return { + loader: 'js', + contents: generateLazyModule(args.path), + } + }) + + build.onLoad({ namespace: 'lazy2', filter: /.*/ }, async args => { + return { + loader: 'js', + contents: generateLazyModule(`lazy3:${args.path}`, false), + } + }) + + build.onLoad({ namespace: rawSeaAssetPrefix.slice(0, -1), filter: /.*/ }, async args => { + const asset = serializerHost?.addRawAsset?.(await fs.readFile(args.path)) + if (!asset) { + return { + loader: 'js', + contents: ``, + } + } + + return { + loader: 'js', + contents: generateRawSeaAsset(asset.hash), + } + }) + + async function readPointer(p: string) { + if (p.length < 100) { + return JSON.parse(await fs.readFile(p, 'utf-8')) as Artifact + } + + const pointer = coerceToPointer(p) + const data = JSON.parse(await fs.readFile(pointer.ref, 'utf-8')) as Artifact + const { storeHash } = pointer.resolve() + if (isNullHash(storeHash)) { + return data + } + + const store = JSON.parse(await fs.readFile(`${pointerPrefix}${storeHash}`, 'utf-8')) + const m = store.type === 'flat' ? store.artifacts[pointer.hash] : undefined + if (!m?.pointers) { + return data + } + + return applyPointers(data, m.pointers) + } + + async function loadPointer(args: esbuild.OnLoadArgs): Promise { + // XXX: we add the prefix back here to make the `esbuild` comments look nicer + const p = `${pointerPrefix}${args.path}` + const unmapped = serializerHost?.getUnmappedPointer?.(p) ?? p + const data = await readPointer(unmapped) + + switch (data.kind) { + case 'compiled-chunk': + return { + loader: 'js', + contents: Buffer.from(data.runtime, 'base64'), + } + case 'deployed': + return { + loader: 'ts', + contents: renderFile(deserializePointers(data), opt.platform, undefined, undefined, false, undefined, serializerHost), + resolveDir: opt.workingDirectory, + pluginData: { virtualId: args.pluginData?.virtualId } + } + default: + throw new Error(`Unknown object kind: ${(data as any).kind}`) + } + } + + build.onLoad({ namespace: 'pointer', filter: /.*/ }, loadPointer) + + build.onLoad({ namespace: 'patched', filter: /.*/ }, async (args) => { + const patchFn = args.pluginData as (contents: string) => string + const contents = await fs.readFile(args.path, 'utf-8') + getLogger().log(`Patching module: ${args.path}`) + + return { + loader: 'js', + contents: patchFn(contents), + } + }) + + build.onLoad({ filter: /\.js$/ }, async (args) => { + const resolved = resolver.getFilePath(args.path) + + return { + loader: 'js', + contents: await fs.readFile(resolved), + } + }) + + build.onLoad({ filter: /\.ts$/ }, async (args) => { + const resolved = resolver.getFilePath(args.path) + + return { + loader: 'ts', + contents: await fs.readFile(resolved), + } + }) + + build.onLoad({ filter: /\.tsx$/ }, async (args) => { + const resolved = resolver.getFilePath(args.path) + + return { + loader: 'tsx', + contents: await fs.readFile(resolved), + } + }) + } + } +} + +export class MemFs { + readonly #memory = new Map() + + public writeFile(fileName: string, content: string) { + this.#memory.set(fileName, content) + } + + public deleteFile(fileName: string) { + this.#memory.delete(fileName) + } + + public fileExists(fileName: string): boolean { + return this.#memory.has(fileName) || ts.sys.fileExists(fileName) + } + + public readFile(fileName: string): string | undefined { + return this.#memory.get(fileName) ?? ts.sys.readFile(fileName) + } + + public getSourceFile(fileName: string, languageVersion: ts.ScriptTarget, onError?: (message: string) => void): ts.SourceFile | undefined { + const sourceText = this.readFile(fileName); + + return sourceText !== undefined + ? ts.createSourceFile(fileName, sourceText, languageVersion) + : undefined; + } + + public print(showContents = false) { + for (const [k, v] of this.#memory.entries()) { + getLogger().log(`file: ${k}`) + if (showContents) { + getLogger().log(v) + } + } + } + + public getFiles() { + return Array.from(this.#memory.keys()) + } + + public *files() { + yield* this.#memory.keys() + } +} + +function createCompilerHost(options: ts.CompilerOptions, fs: SyncFs): ts.CompilerHost { + return { + readFile, + fileExists, + getSourceFile, + getDefaultLibFileName: () => "lib.d.ts", + writeFile: (fileName, content) => fs.writeFileSync(fileName, content), + getCurrentDirectory: () => ts.sys.getCurrentDirectory(), + getDirectories: path => ts.sys.getDirectories(path), + getCanonicalFileName: fileName => ts.sys.useCaseSensitiveFileNames ? fileName : fileName.toLowerCase(), + getNewLine: () => ts.sys.newLine, + useCaseSensitiveFileNames: () => ts.sys.useCaseSensitiveFileNames, + // resolveModuleNames + } + + function fileExists(fileName: string): boolean { + return fs.fileExistsSync(fileName) + } + + function readFile(fileName: string): string | undefined { + if (!fileExists(fileName)) { + return + } + + return fs.readFileSync(fileName, 'utf-8') + } + + function getSourceFile(fileName: string, languageVersion: ts.ScriptTarget, onError?: (message: string) => void) { + const sourceText = readFile(fileName) + + return sourceText !== undefined + ? ts.createSourceFile(fileName, sourceText, languageVersion) + : undefined + } + + // function resolveModuleNames( + // moduleNames: string[], + // containingFile: string + // ): ts.ResolvedModule[] { + // const resolvedModules: ts.ResolvedModule[] = []; + // for (const moduleName of moduleNames) { + // // try to use standard resolution + // let result = ts.resolveModuleName(moduleName, containingFile, options, { + // fileExists, + // readFile, + // }); + // if (result.resolvedModule) { + // resolvedModules.push(result.resolvedModule); + // } else { + // // check fallback locations, for simplicity assume that module at location + // // should be represented by '.d.ts' file + // // for (const location of moduleSearchLocations) { + // // const modulePath = path.join(location, moduleName + ".d.ts"); + // // if (fileExists(modulePath)) { + // // resolvedModules.push({ resolvedFileName: modulePath }); + // // } + // // } + // } + // } + // return resolvedModules; + // } +} + +function generateRawSeaAsset(hash: string) { + return ` +const hash = '${hash}' +const buffer = require('node:sea').getRawAsset(hash) + +module.exports = { buffer, hash } +`.trim() +} + +function generateLazyModule(spec: string, obfuscate = true) { + function obfuscateSpec() { + if (!obfuscate) { + return ts.factory.createStringLiteral(spec) + } + + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('String'), + 'fromCodePoint', + ), + undefined, + [...spec].map(c => c.codePointAt(0)!).map(n => createLiteral(n)) + ) + } + + const loader = ts.factory.createFunctionDeclaration( + undefined, + undefined, + 'load', + undefined, + [ts.factory.createParameterDeclaration(undefined, undefined, 'prop')], + undefined, + ts.factory.createBlock([ + ts.factory.createIfStatement( + ts.factory.createStrictEquality( + ts.factory.createIdentifier('prop'), + ts.factory.createStringLiteral('__esModule') + ), + ts.factory.createReturnStatement(ts.factory.createTrue()) + ), + + createVariableStatement('mod', ts.factory.createCallExpression( + ts.factory.createIdentifier('require'), + undefined, + [obfuscateSpec()] + )), + + ts.factory.createExpressionStatement( + ts.factory.createAssignment( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('module'), + 'exports' + ), + ts.factory.createIdentifier('mod') + ) + ), + + ts.factory.createIfStatement( + ts.factory.createStrictEquality( + ts.factory.createIdentifier('prop'), + ts.factory.createStringLiteral('default') + ), + ts.factory.createReturnStatement(ts.factory.createIdentifier('mod')) + ), + + ts.factory.createReturnStatement( + ts.factory.createElementAccessExpression( + ts.factory.createIdentifier('mod'), + ts.factory.createIdentifier('prop') + ) + ) + ], true) + ) + + const proxyIdent = ts.factory.createIdentifier('p') + const p = ts.factory.createNewExpression( + ts.factory.createIdentifier('Proxy'), + undefined, + [ + ts.factory.createObjectLiteralExpression(), + createObjectLiteral({ + get: ts.factory.createArrowFunction( + undefined, + undefined, + [ + ts.factory.createParameterDeclaration(undefined, undefined, '_'), + ts.factory.createParameterDeclaration(undefined, undefined, 'prop') + ], + undefined, + undefined, + ts.factory.createCallExpression(loader.name!, undefined, [ts.factory.createIdentifier('prop')]) + ), + getPrototypeOf: ts.factory.createArrowFunction( + undefined, + undefined, + [], + undefined, + undefined, + proxyIdent, + ) + }) + ] + ) + + + + const statements = [ + loader, + createVariableStatement(proxyIdent, p), + ts.factory.createExpressionStatement( + ts.factory.createAssignment( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('module'), + 'exports' + ), + proxyIdent + ) + ), + ] + + return printNodes(statements) +} + +export function getNpmDeps(table: Record, manfiest: TerraformPackageManifest) { + const keys = new Set() + for (const v of Object.values(table)) { + if (v.valueType === 'reflection' && v.operations) { + const op = resolveVal(table, v.operations[0]) + if (op.type !== 'import') continue + + keys.add(op.module) + } + } + + return pruneManifest(manfiest, keys) +} + +function resolveVal(table: Record, val: T): T { + if (moveableStr in val) { + const id = (val as any)[moveableStr].id + const obj = table[id] + if (obj.valueType !== 'object') { + throw new Error(`Invalid value type: ${obj.valueType}. Expected type "object".`) + } + + const res: Record = {} + for (const [k, v] of Object.entries((obj as any).properties)) { + if (typeof v === 'object' && !!v) { + res[k] = resolveVal(table, v as any) + } else { + res[k] = v + } + } + return res as T + } + + return val +} + +// FIXME: `esm` not implemented +function createLoader( + table: Record, + isBundled: boolean, + type: 'cjs' | 'esm' = 'cjs', + platform: 'node' | 'browser' = 'node', + factory = ts.factory +) { + const name = factory.createIdentifier('loadModule') + const targetIdent = factory.createIdentifier('target') + const originIdent = factory.createIdentifier('origin') + + if (!isBundled) { + const targetParameter = factory.createParameterDeclaration(undefined, undefined, targetIdent, undefined, undefined, undefined) + const returnRequire = factory.createReturnStatement( + factory.createCallExpression( + factory.createIdentifier('require'), + undefined, + [targetIdent] + ) + ) + + const body = factory.createBlock([returnRequire], true) + + return factory.createFunctionDeclaration(undefined, undefined, name, undefined, [targetParameter], undefined, body) + } + + const defaultClause = factory.createDefaultClause([ + factory.createThrowStatement( + factory.createNewExpression( + factory.createIdentifier('Error'), + undefined, + [factory.createTemplateExpression( + factory.createTemplateHead('No module found for target: '), + [ + factory.createTemplateSpan( + targetIdent, + factory.createTemplateMiddle(' (') + ), + factory.createTemplateSpan( + originIdent, + factory.createTemplateTail(')') + ) + ] + )] + ) + ) + ]) + + const defaultCases: Record = {} + const originCases: Record = {} + for (const entry of Object.values(table)) { + switch (entry?.valueType) { + case 'function': { + const spec = isDataPointer(entry.module) + ? entry.module.ref + : entry.module.startsWith(pointerPrefix) ? entry.module : entry.module + + const resolved = factory.createStringLiteral(spec) + const clause = factory.createCaseClause( + factory.createStringLiteral(spec), + [factory.createReturnStatement( + factory.createCallExpression( + factory.createIdentifier('require'), + undefined, + [resolved] + ) + )] + ) + + defaultCases[spec] = clause + + break + } + case 'reflection': { + const firstOp = resolveVal(table, entry.operations![0]) + + if (firstOp.type !== 'import') continue + + // const packageName = (entry as any).packageName + // const resolved = !isBuiltin(firstOp.module) && packageName + // ? createRequire(require.resolve(packageName)).resolve(firstOp.module) + // : firstOp.module + + const clause = factory.createCaseClause( + factory.createStringLiteral(firstOp.module), + [factory.createReturnStatement( + factory.createCallExpression( + factory.createIdentifier('require'), + undefined, + [factory.createStringLiteral(firstOp.module)] + ) + )] + ) + + //if (!packageName) { + defaultCases[firstOp.module] = clause + // } else { + // originCases[v.origin] ??= { clauses: [] } + // originCases[v.origin].clauses.push(clause) + // } + + break + } + } + } + + const outerCases: ts.CaseClause[] = [] + for (const [k, v] of Object.entries(originCases)) { + const caseBlock = factory.createCaseBlock([...v.clauses, defaultClause]) + const switchStatement = factory.createSwitchStatement(targetIdent, caseBlock) + + const clause = factory.createCaseClause( + factory.createStringLiteral(k), + [switchStatement] + ) + outerCases.push(clause) + } + + const defaultOriginBlock = factory.createCaseBlock([...Object.values(defaultCases), defaultClause]) + const defaultOriginClause = factory.createDefaultClause([ + factory.createSwitchStatement(targetIdent, defaultOriginBlock) + ]) + + const caseBlock = factory.createCaseBlock([...outerCases, defaultOriginClause]) + const switchStatement = factory.createSwitchStatement(originIdent, caseBlock) + const targetParameter = factory.createParameterDeclaration(undefined, undefined, targetIdent, undefined, undefined, undefined) + const originParameter = factory.createParameterDeclaration(undefined, undefined, originIdent, undefined, undefined, undefined) + + const body = factory.createBlock([switchStatement], true) + + return factory.createFunctionDeclaration(undefined, undefined, name, undefined, [targetParameter, originParameter], undefined, body) +} + +function createResolver( + loader: ts.Identifier, + dataTable: Record | string, + factory = ts.factory +) { + const req = factory.createCallExpression( + factory.createIdentifier('require'), + undefined, + [factory.createStringLiteral('synapse:serdes')] + ) + + const resolve = factory.createPropertyAccessExpression( + req, + 'resolveValue' + ) + + const valueIdent = factory.createIdentifier('value') + const valueParameter = factory.createParameterDeclaration(undefined, undefined, valueIdent, undefined, undefined, undefined) + const body = factory.createCallExpression( + resolve, + undefined, + [ + valueIdent, + createObjectLiteral({ loadModule: loader }, factory), + typeof dataTable === 'object' + ? createObjectLiteral(dataTable as any, factory) + : factory.createCallExpression( + factory.createPropertyAccessExpression(factory.createIdentifier('JSON'), 'parse'), + undefined, + [factory.createStringLiteral(dataTable)] // stringified + ) + ] + ) + + return createVariableStatement( + 'resolveValue', + factory.createArrowFunction(undefined, undefined, [valueParameter], undefined, undefined, body) + ) +} + +export function serializePointers(obj: any): any { + if (typeof obj !== 'object' || !obj) { + return obj + } + + if (Array.isArray(obj)) { + return obj.map(serializePointers) + } + + if (isDataPointer(obj)) { + const { hash, storeHash } = obj.resolve() + + return { [moveableStr]: { valueType: 'data-pointer', hash, storeHash } } + } + + const res: Record = {} + for (const [k, v] of Object.entries(obj)) { + res[k] = serializePointers(v) + } + + return res +} + +function deserializePointers(obj: any): any { + if (typeof obj !== 'object' || !obj) { + return obj + } + + if (Array.isArray(obj)) { + return obj.map(deserializePointers) + } + + if (moveableStr in obj && obj[moveableStr].valueType === 'data-pointer') { + return createPointer(obj[moveableStr].hash, obj[moveableStr].storeHash) + } + + if (isDataPointer(obj)) { + return obj + } + + const res: Record = {} + for (const [k, v] of Object.entries(obj)) { + res[k] = deserializePointers(v) + } + + return res +} + +interface SerializerHost { + addAsset?: (p: DataPointer) => string + addRawAsset?: (data: ArrayBuffer) => DataPointer + getMappedPointer?: (p: string) => string + getUnmappedPointer?: (p: string) => string + optimize?: Optimizer +} + +export function renderFile( + data: { table: Record; captured: any }, + platform: 'node' | 'browser' = 'node', + isBundled = true, + immediatelyInvoke = false, + useExportDefault = isBundled, + shouldSerializePointers = false, + host?: Pick +) { + if (isBundled) { + if (host?.optimize) { + data = host.optimize(data.table, data.captured) + } + return printNodes(renderSerializedData(data.table, data.captured, platform, immediatelyInvoke, host)) + } + + // Dead code? + + const loader = createLoader(data.table, isBundled, 'cjs', platform) + const { table, captured } = shouldSerializePointers ? serializePointers(data) : data + const resolver = createResolver(ts.factory.createIdentifier(loader.name!.text), table) + const resolved = ts.factory.createCallExpression( + ts.factory.createIdentifier('resolveValue'), + undefined, + [createObjectLiteral(captured, ts.factory)] + ) + + const sourceFile = printNodes([ + loader, + resolver, + immediatelyInvoke + ? ts.factory.createCallExpression(resolved, undefined, []) + : useExportDefault + ? ts.factory.createExportDefault(resolved) + : ts.factory.createExportAssignment(undefined, true, resolved) + ]) + + return sourceFile +} + +const fn = (body: ts.Expression | ts.Block) => ts.factory.createArrowFunction(undefined, undefined, [], undefined, undefined, body) +let noop: ts.ArrowFunction +let emptyClass: ts.ClassExpression + +function tryOptimization(ops: ReflectionOperation[]) { + if (ops.length < 2 || ops[0].type !== 'import') { + return + } + + noop ??= fn(ts.factory.createBlock([])) + emptyClass ??= ts.factory.createClassExpression(undefined, undefined, undefined, undefined, []) + + if (ops[0].module.startsWith('synapse:')) { + if (ops.length === 2) { + if (ops[1].type === 'get' && ops[1].property === 'getContext') { + return fn(createObjectLiteral({}, ts.factory)) + } + + if (ops[1].type === 'get' && ops[1].property === 'getCurrentId') { + return noop + } + + if (ops[1].type === 'get' && ops[1].property === 'generateName2') { + return noop + } + + if (ops[1].type === 'get' && ops[1].property === 'generateIdentifier') { + return noop + } + + if (ops[1].type === 'get' && ops[1].property === 'requireSecret') { + return noop + } + + if (ops[1].type === 'get' && ops[1].property === 'getLogger') { + return fn(ts.factory.createIdentifier('console')) + } + + if (ops[1].type === 'get' && ops[1].property === 'Fn') { + return createObjectLiteral({}, ts.factory) + } + + if (ops[1].type === 'get' && ops[1].property === 'defineResource') { + return fn(emptyClass) + } + + if (ops[1].type === 'get' && ops[1].property === 'Bundle') { + return emptyClass + } + + if (ops[1].type === 'get' && ops[1].property === 'Archive') { + return emptyClass + } + } else if (ops.length === 3) { + if (ops[1].type === 'get' && ops[1].property === 'defineResource' && ops[2].type === 'apply') { + return emptyClass + } + + if (ops[1].type === 'get' && ops[1].property === 'defineDataSource' && ops[2].type === 'apply') { + return fn(noop) + } + } + } +} + +function renderSerializedData( + table: Record, + captured: any, + platform: 'node' | 'browser' | 'synapse', + immediatelyInvoke?: boolean, + host?: SerializerHost +) { + const addAsset = host?.addAsset + const getMappedPointer = host?.getMappedPointer + + const imports: ts.Node[] = [] + const required = new Set() // All object ids that are needed + const statements = new Map() + const dependencies = new Map() + const extraStatements = new Map() + + function addExtraStatements(id: number | string, statements: ts.Node[]) { + if (!extraStatements.has(id)) { + extraStatements.set(id, []) + } + + extraStatements.get(id)!.push(...statements) + } + + function createIdent(id: number | string) { + if (typeof id === 'string' && id.startsWith('b:')) { // Bound mutable + const ident = ts.factory.createIdentifier(`nb_${id.slice(2)}`) + + return ident + } + + return ts.factory.createIdentifier(`n_${id}`) + } + + const importIdents = new Map() + function createImport(spec: string, isNamespaceImport?: boolean, member?: string) { + if (spec.startsWith(pointerPrefix) && getMappedPointer) { + spec = getMappedPointer(spec) + } + + const key = member ? `${spec}#${member}` : spec + if (importIdents.has(key)) { + return importIdents.get(key)! + } + + const ident = ts.factory.createIdentifier(`import_${importIdents.size}`) + importIdents.set(key, ident) + + const importClause = member !== undefined + ? ts.factory.createImportClause(false, undefined, ts.factory.createNamedImports([ + ts.factory.createImportSpecifier(false, ts.factory.createIdentifier(member), ident) + ])) + : isNamespaceImport + ? ts.factory.createImportClause(false, undefined, ts.factory.createNamespaceImport(ident)) + : ts.factory.createImportClause(false, ident, undefined) + + imports.push(ts.factory.createImportDeclaration( + undefined, + importClause, + ts.factory.createStringLiteral(spec) + )) + + return ident + } + + function renderReflection(operations: ReflectionOperation[], id?: number | string) { + // Need to expand ops (this is a sign of brittleness) + operations = operations.map(o => { + if (moveableStr in o) { + return (table[(o as any)[moveableStr].id] as any).properties as ReflectionOperation + } + + return o + }) + + const optimized = tryOptimization(operations) + if (optimized) { + if (id !== undefined) { + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, optimized)) + + return ident + } + + return optimized + } + + let currentNode: ts.Expression + for (let i = 0; i < operations.length; i++) { + const op = operations[i] + switch (op.type) { + case 'import': { + // Handles `import { foo } from 'bar'` + // const nextOp = operations[i+1] + // if (nextOp?.type === 'get') { + // i += 1 + // currentNode = createImport(op.module, undefined, nextOp.property) + // break + // } + currentNode = createImport(op.module, true) + break + } + case 'global': { + currentNode = ts.factory.createIdentifier('globalThis') + break + } + case 'get': { + currentNode = ts.factory.createElementAccessExpression( + currentNode!, + ts.factory.createStringLiteral(op.property) + ) + break + } + case 'apply': { + // TODO: use `apply` and pass in `thisArg` if present? + currentNode = ts.factory.createCallExpression( + currentNode!, + undefined, + op.args.map(render) + ) + break + } + case 'construct': { + currentNode = ts.factory.createNewExpression( + currentNode!, + undefined, + op.args.map(render) + ) + break + } + } + } + + if (id !== undefined) { + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, currentNode!)) + + return ident + } + + return currentNode! + } + + function renderBoundFunction(id: number | string, boundTarget: any, boundThisArg: any, boundArgs: any[]) { + const bound = ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression(render(boundTarget), 'bind'), + undefined, + [render(boundThisArg), ...boundArgs.map(render)] + ) + + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, bound)) + + return ident + } + + function renderFunction(id: number | string, module: string, args: any[]) { + const spec = isDataPointer(module) + ? toAbsolute(module) + : module.startsWith(pointerPrefix) ? module : `${pointerPrefix}${module}` + + const fn = createImport(spec) + const exp = ts.factory.createCallExpression(fn, undefined, args.map(render)) + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, exp)) + + return ident + } + + function renderObject(id: number | string, properties?: Record, constructor?: any, descriptors?: any) { + const proto = constructor !== undefined + ? ts.factory.createPropertyAccessExpression(render(constructor), 'prototype') + : ts.factory.createNull() + + const createExp = ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Object'), + 'create' + ), + undefined, + descriptors ? [proto, render(descriptors)] : [proto] + ) + + const assignExp = ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Object'), + 'assign' + ), + undefined, + properties && Object.keys(properties).length > 0 + ? [createExp, render(properties as any)] + : [createExp] + ) + + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, assignExp)) + + return ident + } + + function renderLiteral(obj: any): ts.Expression { + if (typeof obj !== 'object' || obj === null) { + return createLiteral(obj) + } + + if (Array.isArray(obj)) { + return createArrayLiteral(ts.factory, obj.map(render)) + } + + if (moveableStr in obj) { + const v = obj[moveableStr] + if ('id' in v && Object.keys(v).length === 1) { + frame.push(v.id) + + return createIdent(v.id) + } + + return render(v) + } + + if (isDataPointer(obj)) { + if (addAsset) { + return createLiteral(addAsset(obj)) + } + return createLiteral(obj.ref) + } + + const res: Record = {} + for (const [k, v] of Object.entries(obj)) { + res[k] = render(v as any) + } + + return createObjectLiteral(res, ts.factory) + } + + function renderRegExp(id: number | string, source: string, flags: string) { + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, ts.factory.createRegularExpressionLiteral(`/${source}/${flags}`))) + + return ident + } + + function renderResource(id: number | string, value: any) { + const normalized = normalizeTerraform(value) + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, renderLiteral(normalized))) + + return ident + } + + function renderBinding(id: number | string, key: string, value: number | string | { [moveableStr]: { id: number } }) { + const ident = createIdent(id) + statements.set(id, createVariableStatement(ident, createObjectLiteral({}, ts.factory))) + + const assignmentId = `${id}_a` + const valueId = typeof value === 'object' ? value['@@__moveable__']['id'] : value + const assignment = ts.factory.createAssignment( + ts.factory.createPropertyAccessExpression(ident, key), + createIdent(valueId) + ) + + statements.set(assignmentId, assignment) + required.add(assignmentId) + dependencies.set(assignmentId, [id, valueId]) + + return ident + } + + function renderDataPointer(hash: string, storeHash: string): ts.Expression { + throw new Error(`Rendering data pointer is not implemented`) + } + + // TODO: support rendering things inline instead of only by reference + function render(obj: ExternalValue): ts.Expression { + switch (obj?.valueType) { + case 'reflection': + return renderReflection(obj.operations!, obj.id) + case 'function': + return renderFunction(obj.id!, obj.module, obj.captured!) + case 'bound-function': + return renderBoundFunction(obj.id!, obj.boundTarget, obj.boundThisArg, obj.boundArgs!) + case 'regexp': + return renderRegExp(obj.id!, (obj as any).source, (obj as any).flags) + case 'resource': + return renderResource(obj.id!, (obj as any).value) + case 'object': + return renderObject( + obj.id!, + (obj as any).properties, + Object.prototype.hasOwnProperty.call(obj, 'constructor') ? (obj as any).constructor : undefined, + (obj as any).descriptors + ) + case 'binding': + return renderBinding(obj.id!, obj.key!, obj.value!) + // case 'data-pointer': + // return renderDataPointer(obj.hash!, (obj as any).storeHash) + default: + return renderLiteral(obj) + } + } + + let frame: (string | number)[] + function renderEntry(entry: ExternalValue) { + frame = [] + dependencies.set(entry.id!, frame) + const exp = render({ + ...entry, + // Strip out permissions and alt. implementations + symbols: entry.symbols ? { + ...entry.symbols, + browserImpl: undefined, + permissions: undefined, + } : undefined + }) + + const objectId = getSymbol(entry, 'synapse.objectId') + if (objectId !== undefined && ts.isIdentifier(exp)) { + addExtraStatements(entry.id!, [ + ts.factory.createAssignment( + ts.factory.createElementAccessExpression(exp, createSymbolPropertyName('synapse.objectId')), + renderLiteral(objectId) + ) + ]) + } + } + + function getSymbol(val: ExternalValue, name: string) { + if (!val.symbols) return + + if (moveableStr in val.symbols) { + const resolved = table[val.symbols[moveableStr].id] + if (resolved?.valueType === 'object') { + return (resolved as any).properties[name] + } + + return + } + + return val.symbols[name] + } + + for (const obj of Object.values(table)) { + if (obj === null || obj === undefined) continue // Not needed anymore?? + + const browserImpl = platform === 'browser' ? getSymbol(obj, 'browserImpl') : undefined + if (browserImpl) { + const resolved = moveableStr in browserImpl ? table[browserImpl[moveableStr].id] : browserImpl + renderEntry({ ...resolved, id: obj.id }) + } else { + renderEntry(obj) + } + } + + const edges = Array.from(dependencies.entries()).flatMap(([k, v]) => v.map(x => [k, x] as const)) + const sorted = topoSort(edges as [number, number][]) + + // Dead code elimination + function visit(obj: any): void { + if (typeof obj !== 'object' || obj === null) { + return + } + + if (Array.isArray(obj)) { + return obj.forEach(visit) + } + + for (const [k, v] of Object.entries(obj)) { + if (k === moveableStr && 'id' in (v as any)) { + required.add((v as any).id) + } else { + visit(v) + } + } + } + + visit(captured) + const rootNodes = new Set(required) + for (const id of sorted) { + if (required.has(id)) { + if (!dependencies.has(id)) { + throw new Error(`Missing entry for ref: ${id} - ${JSON.stringify(table[id])}`) + } + + dependencies.get(id)!.forEach(x => required.add(x)) + } + } + + const pruned = sorted.filter(x => required.has(x)).reverse() as (string | number)[] + + // Some root nodes may not be in the topo sort because they have no edges + const prunedSet = new Set(pruned) + for (const id of rootNodes) { + if (!prunedSet.has(id)) { + pruned.push(id) + prunedSet.add(id) + } + } + + function createEsmExport(obj: any) { + const specs = Object.entries(obj) + .filter(x => typeof x[1] === 'object') + .map(([k, v]) => { + return ts.factory.createExportSpecifier( + false, + render(v as any) as ts.Identifier, + k + ) + }) + + const namedExports = ts.factory.createNamedExports(specs) + + return ts.factory.createExportDeclaration(undefined, false, namedExports) + } + + function createExportsDeclaration(obj: any) { + if (typeof obj !== 'object' || !obj || Array.isArray(obj) || moveableStr in obj) { + if (moveableStr in obj) { + const id = obj[moveableStr].id + if (id !== undefined) { + const val = table[id] + if (val.valueType !== 'object') { + return ts.factory.createExportDefault(render(obj)) + } + + if ((val as any).properties.__esModule) { + const bindings = (val as any).properties + delete bindings.__esModule + return createEsmExport(bindings) + } + } + } + + return ts.factory.createExportDefault(render(obj)) + } + + return createEsmExport(obj) + } + + return [ + ...imports, + ...pruned.flatMap(id => { + const statement = statements.get(id)! + const extras = extraStatements.get(id) + if (!extras) { + return statement + } + + return [statement, ...extras] + }), + immediatelyInvoke + ? ts.factory.createCallExpression( + render(captured), + undefined, + [] + ) + : createExportsDeclaration(captured) + ] +} + +const capitalize = (s: string) => s ? s.charAt(0).toUpperCase().concat(s.slice(1)) : s +function normalize(str: string) { + const [first, ...rest] = str.split('_') + + return [first, ...rest.map(capitalize)].join('') +} + +function normalizeTerraform(obj: any): any { + if (typeof obj !== 'object' || !obj) { + return obj + } + + if (Array.isArray(obj)) { + return obj.map(normalizeTerraform) + } + + if (isDataPointer(obj)) { + return obj + } + + const res: Record = {} + for (const [k, v] of Object.entries(obj)) { + // Don't normalize everything + if (k === moveableStr) { + res[k] = v + } else { + res[normalize(k)] = normalizeTerraform(v) + } + } + + return res +} + +const moveableStr = '@@__moveable__' + diff --git a/src/cli/buildInternal.ts b/src/cli/buildInternal.ts new file mode 100644 index 0000000..d118220 --- /dev/null +++ b/src/cli/buildInternal.ts @@ -0,0 +1,925 @@ +import * as path from 'node:path' +import * as zlib from 'node:zlib' +import * as github from '../utils/github' +import { getGlobalCacheDirectory, getUserSynapseDirectory, resolveProgramBuildTarget } from '../workspaces' +import { getBuildTargetOrThrow, getFs } from '../execution' +import { createTarball, extractFileFromZip, extractTarball, hasBsdTar } from '../utils/tar' +import { PackageJson, getPackageJson } from '../pm/packageJson' +import { downloadSource } from '../build/sources' +import { buildGoProgram } from '../build/go' +import { installModules } from '../pm/packages' +import { createMergedView } from '../pm/publish' +import { Snapshot, consolidateBuild, createSnapshot, dumpData, getDataRepository, getModuleMappings, getProgramFs, linkFs, pruneBuild, writeSnapshotFile } from '../artifacts' +import { QualifiedBuildTarget, resolveBuildTarget } from '../build/builder' +import { runCommand } from '../utils/process' +import { toAbsolute, toDataPointer } from '../build-fs/pointers' +import { glob } from '../utils/glob' +import { gzip, memoize, throwIfNotFileNotFoundError } from '../utils' +import { getLogger } from '..' +import { randomUUID } from 'node:crypto' +import { createZipFromDir } from '../deploy/deployment' + + +const integrations = { + 'synapse-aws': 'integrations/aws', + 'synapse-local': 'integrations/local', + + // Frontend stuff + 'synapse-react': 'integrations/frontend-runtimes/react', + 'synapse-websites': 'integrations/websites', +} + +export async function copyIntegrations(rootDir: string, dest: string, included?: string[]) { + const packagesDir = path.resolve(dest, 'packages') + const include = included ? new Set(included) : undefined + for (const [k, v] of Object.entries(integrations)) { + if (include && !include.has(k)) continue + const integrationPkgPath = path.resolve(rootDir, v) + await createPackageForRelease(integrationPkgPath, path.resolve(packagesDir, k), undefined, true) + } +} + +const baseUrl = 'https://nodejs.org/download/release' + +// Needed when using `musl` +const unofficialUrl = 'https://unofficial-builds.nodejs.org/download/release' + +function getDownloadUrl(version: string, target: QualifiedBuildTarget) { + const archSuffix = target.arch === 'aarch64' ? 'arm64' : target.arch + const os = target.os === 'windows' ? 'win' : target.os + const extname = target.os === 'windows' ? '.zip' : '.tar.gz' + const libc = target.libc + const name = ['node', version, os, `${archSuffix}${libc ? `-${libc}` : ''}${extname}`].join('-') + const url = !libc ? baseUrl : unofficialUrl + + return `${url}/${version}/${name}` +} + +function decompress(data: Buffer, format: 'bz' | 'gz') { + if (format === 'gz') { + return new Promise((resolve, reject) => { + zlib.gunzip(data, (err, res) => err ? reject(err) : resolve(res)) + }) + } + + return new Promise((resolve, reject) => { + zlib.brotliDecompress(data, (err, res) => err ? reject(err) : resolve(res)) + }) +} + +const getNodeBinCacheDir = () => path.resolve(getGlobalCacheDirectory(), 'node') + +const doReq = (url: string) => new Promise((resolve, reject) => { + const https = require('node:https') as typeof import('node:https') + const req = https.request(url, { method: 'GET' }, resp => { + const buffer: any[] = [] + resp.on('data', d => buffer.push(d)) + resp.on('end', () => { + if (!resp.statusCode) { + return reject(new Error('Response contained no status code')) + } + + if (resp.statusCode >= 400) { + return reject(Object.assign(new Error(buffer.join('')), { statusCode: resp.statusCode })) + } + + if (resp.headers['content-type'] === 'application/json') { + resolve(JSON.parse(buffer.join(''))) + } else { + resolve(Buffer.concat(buffer)) + } + }) + resp.on('error', reject) + }) + + req.end() +}) + +async function listFilesInZip(zip: Buffer) { + if (!(await hasBsdTar())) { + const tmp = path.resolve(process.cwd(), 'dist', `tmp-${randomUUID()}.zip`) + await getFs().writeFile(tmp, zip) + const res = await runCommand('unzip', ['-l', tmp]).finally(async () => { + await getFs().deleteFile(tmp) + }) + + // first three lines and last two lines we don't care + const lines = res.trim().split('\n').slice(3, -2) + return lines.map(l => { + const [length, date, time, name] = l.trim().split(/\s+/) + + return name + }) + } + + if (process.platform === 'win32') { + const tmp = path.resolve(process.cwd(), 'dist', `tmp-${randomUUID()}.zip`) + await getFs().writeFile(tmp, zip) + const res = await runCommand('tar', ['-tzf', tmp]).finally(async () => { + await getFs().deleteFile(tmp) + }) + + return res.split(/\r?\n/).map(x => x.trim()).filter(x => !!x) + } + + // Only works with `bsdtar` + const res = await runCommand('tar', ['-tzf-'], { + input: zip, + }) + + return res.split(/\r?\n/).map(x => x.trim()).filter(x => !!x) +} + +async function getOrDownloadNode(version: string, target: QualifiedBuildTarget) { + const p = path.resolve(getNodeBinCacheDir(), `${version}-${target.os}-${target.arch}${target.libc ? `-${target.libc}` : ''}`) + if (await getFs().fileExists(p)) { + return p + } + + const url = getDownloadUrl(version, target) + const d = await doReq(url) + if (target.os === 'windows') { + const prefix = path.basename(url).replace(/.zip$/, '') + const executable = await extractFileFromZip(d, `${prefix}/node.exe`) + await getFs().writeFile(p, executable) + + return p + } + + const tarball = extractTarball(await decompress(d, 'gz')) + const executable = tarball.find(x => x.path.endsWith('bin/node')) + if (!executable) { + throw new Error(`Failed to find executable in tarball: ${tarball.map(x => x.path)}`) + } + + // TODO: check signature + integrity + await getFs().writeFile(p, executable.contents) + + return p +} + +async function getPackageJsonOrThrow(pkgDir: string) { + const pkg = await getPackageJson(getFs(), pkgDir, false) + if (!pkg) { + throw new Error(`Missing package.json: ${pkgDir}`) + } + return pkg +} + +// TODO: turn this into a resource that resolves into a relative path +async function getNodeJsForPkg(pkgDir: string, target?: Partial) { + const resolved = resolveBuildTarget(target) + const pkg = await getPackageJsonOrThrow(pkgDir) + + const nodeEngine = pkg.data.engines?.node + if (!nodeEngine) { + throw new Error(`No node version found: ${pkgDir}`) + } + + const nodePath = await getOrDownloadNode(`v${nodeEngine}`, resolved) + await getFs().writeFile( + path.resolve(pkgDir, 'bin', target?.os === 'windows' ? 'node.exe' : 'node'), + await getFs().readFile(nodePath) + ) +} + +export async function downloadNodeLib(owner = 'Cohesible', repo = 'node') { + const dest = path.resolve('dist', 'node.lib') + if (await getFs().fileExists(dest)) { + return dest + } + + const assetName = 'node-lib-windows-x64' + + async function downloadAndExtract(url: string) { + const archive = await github.fetchData(url) + const files = await listFilesInZip(archive) + if (files.length === 0) { + throw new Error(`Archive contains no files: ${url}`) + } + + const file = await extractFileFromZip(archive, files[0]) + await getFs().writeFile(path.resolve('dist', 'node.lib'), file) + getLogger().log('Downloaded node.lib to dist/node.lib') + + return dest + } + + if (repo === 'node') { + const release = await github.getRelease(owner, repo) + const asset = release.assets.find(a => a.name === `${assetName}.zip`) + if (!asset) { + throw new Error(`Failed to find "${assetName}" in release "${release.name} [tag: ${release.tag_name}]"`) + } + + return downloadAndExtract(asset.browser_download_url) + } + + const artifacts = (await github.listArtifacts(owner, repo)).sort( + (a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime() + ) + + const match = artifacts.find(a => a.name === assetName) + if (!match) { + return + } + + return downloadAndExtract(match.archive_download_url) +} + +async function maybeUseGithubArtifact(ref: string, target: QualifiedBuildTarget, name: string) { + const parsed = github.parseDependencyRef(ref) + if (parsed.type !== 'github') { + throw new Error(`Not implemented: ${parsed.type}`) + } + + const artifacts = (await github.listArtifacts(parsed.owner, parsed.repository)).sort( + (a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime() + ) + + // TODO: consolidate os/arch normalization + const arch = target.arch === 'aarch64' ? 'arm64' : target.arch === 'x64' ? 'amd64' : target.arch + const match = artifacts.find(a => a.name.endsWith(`${name}-${target.os}-${arch}`)) + ?? artifacts.find(a => a.name.endsWith(`${name}-${target.os}-${target.arch}`)) + + if (!match) { + return + } + + const archive = await github.fetchData(match.archive_download_url) + + const files = await listFilesInZip(archive) + if (files.length === 0) { + throw new Error(`Archive contains no files: ${match.archive_download_url}`) + } + + const file = await extractFileFromZip(archive, files[0]) + + return file +} + +interface NodeBuildOptions { + lto?: boolean +} + +async function findLibtoolFromClang(clangPath: string) { + const rp = (await runCommand(`realpath`, [clangPath])).trim() + const res = path.resolve(rp, '..', 'llvm-libtool-darwin') + if (!(await getFs().fileExists(res))) { + return + } + + return res +} + +const homebrewClangPath = '/opt/homebrew/bin/clang' + +// Needs python + ninja installed +async function buildCustomNodeBin(source: string, target?: Partial & NodeBuildOptions) { + const configArgs = ['--without-npm', '--without-corepack', '--ninja'] + if (target?.lto) { + configArgs.push('--enable-lto') + } + + const env = { ...process.env } + if (!env.CC && await getFs().fileExists(homebrewClangPath)) { + env.CC = homebrewClangPath + env.LIBTOOL ??= await findLibtoolFromClang(env.CC) + } + + if (!env.CXX && await getFs().fileExists(`${homebrewClangPath}++`)) { + env.CXX = `${homebrewClangPath}++` + } + + // TODO: delete `config.status` if CC/CXX/LIBTOOL changes + + function isSameAsConfig(args: string[]) { + if (args.length !== configArgs.length) { + return false + } + + args.sort() + configArgs.sort() + for (let i = 0; i < args.length; i++) { + if (args[i] !== configArgs[i]) { + return false + } + } + + return true + } + + if (!(await getFs().fileExists(path.resolve(source, 'config.status')))) { + getLogger().log('Configuring NodeJS build') + await runCommand(path.resolve(source, 'configure'), configArgs, { cwd: source, env }) + } else { + const statusFile = await getFs().readFile(path.resolve(source, 'config.status'), 'utf-8') + const args = statusFile.match(/exec \.\/configure (.*)/)?.[1]?.split(' ') + if (!args || !isSameAsConfig(args)) { + await runCommand(path.resolve(source, 'configure'), configArgs, { + cwd: source, + env, + }) + } + } + + getLogger().log('Building custom NodeJS binary') + await runCommand('make', ['-j16'], { cwd: source, env }) + const outPath = path.resolve(source, 'out', 'Release', 'node') + + return outPath +} + +// TODO: turn this into a resource that resolves into `Record` where the value is a relative path +export async function buildBinaryDeps(pkgDir: string, target?: Partial & { snapshot?: boolean; downloadOnly?: boolean }) { + const resolved = resolveBuildTarget(target) + const pkg = await getPackageJsonOrThrow(pkgDir) + + const res: Record = {} + const deps = pkg.data.synapse?.binaryDependencies + if (!deps) { + return res + } + + const binDir = path.resolve(pkgDir, 'bin') + const toolsDir = path.resolve(pkgDir, 'tools') + for (const [k, v] of Object.entries(deps)) { + const isNode = k === 'node' + if (isNode && !target?.snapshot && !target?.downloadOnly) continue + + const outputName = target?.os === 'windows' ? `${k}.exe` : k + + const artifact = await maybeUseGithubArtifact(v, resolved, k === 'terraform' ? `${k}-1.5.5` : k).catch(e => { + if (target?.downloadOnly) { + throw e + } + + getLogger().log(`Failed to get artifact from "${v}"`, e) + }) + + const dest = isNode ? path.resolve(binDir, outputName) : path.resolve(toolsDir, outputName) + + if (artifact) { + getLogger().log(`Using pre-built artifact for dependency: ${k}`) + res[k] = dest + + await getFs().writeFile(dest, artifact) + + continue + } + + if (target?.downloadOnly) { + throw new Error(`No artifact found: ${k}`) + } + + const source = await downloadSource({ + type: 'git', + url: v, + commitish: 'main', + }) + + if (isNode) { + const outpath = await buildCustomNodeBin(source, target) + await getFs().deleteFile(path.resolve(binDir, outputName)).catch(throwIfNotFileNotFoundError) + await getFs().writeFile( + path.resolve(binDir, outputName), + await getFs().readFile(outpath) + ) + continue + } + + res[k] = dest + await buildGoProgram({ + sourceDir: source, + output: dest, + target: { + mode: 'release', + os: resolved.os, + arch: resolved.arch, + } + }) + } + + return res +} + +interface BuildTargetExtras { + external?: string[] + sign?: boolean + skipPackage?: boolean + stripInternal?: boolean + buildLicense?: boolean + + // For building nodejs + lto?: boolean + snapshot?: boolean + downloadOnly?: boolean + environmentName?: string +} + +export async function createPackageForRelease(pkgDir: string, dest: string, target?: Partial & BuildTargetExtras, isIntegration?: boolean, useCompiledPkgJson = false) { + const pkg = await getPackageJsonOrThrow(pkgDir) + const bt = await resolveProgramBuildTarget(pkgDir, { environmentName: target?.environmentName }) + if (!bt) { + throw new Error(`Failed to resolve build target: ${pkgDir}`) + } + + function removeExternalDeps(external: string[], pkgData: PackageJson) { + if (!pkgData.dependencies && !pkgData.devDependencies) { + return pkgData + } + + const copy = { ...pkgData, dependencies: { ...pkgData.dependencies } } + for (const [k, v] of Object.entries(copy.dependencies)) { + if (external.find(x => k.startsWith(x))) { + delete copy.dependencies[k] + } + } + + delete copy.devDependencies + delete copy.scripts + + return copy + } + + const fs = getFs() + + const programId = bt.programId + const deploymentId = bt.deploymentId + + const programFs = getProgramFs(programId) + const filesToKeep: string[] = [] + + const moduleMappings = await getModuleMappings(programFs) + if (moduleMappings) { + for (const [k, v] of Object.entries(moduleMappings)) { + filesToKeep.push(v.path) + filesToKeep.push(v.path.replace(/\.js$/, '.infra.js')) + } + } + + const pruned = await createMergedView(programId, deploymentId) + + for (const f of Object.keys(pruned.files)) { + // XXX: dump all `.wasm` and `.node` files to `dist` + if (f.endsWith('.wasm') || f.endsWith('.node')) { + await fs.writeFile( + path.resolve(dest, 'dist', path.basename(f)), + await programFs.readFile(f) + ) + } + + if (isIntegration && (f.endsWith('.js') || f.endsWith('.d.ts')) || f === 'package.json') { + filesToKeep.push(f) + } + } + + const consolidated = await consolidateBuild(getDataRepository(), pruned, filesToKeep, { strip: true }) + const { snapshot } = await createSnapshot(consolidated.index, programId, deploymentId) + + pruneSnapshot(snapshot, filesToKeep, target?.stripInternal) + + // Remap `pointers` + for (const v of Object.values(snapshot.pointers)) { + for (const [k, p] of Object.entries(v)) { + v[k] = toAbsolute(consolidated.copier!.getCopiedOrThrow(p)) + } + } + + // Same as `pointers` + if (snapshot.published && isIntegration) { + for (const [k, p] of Object.entries(snapshot.published)) { + snapshot.published[k] = toAbsolute(consolidated.copier!.getCopiedOrThrow(`pointer:${p}`)).slice('pointer:'.length) + } + } + + await writeSnapshotFile(fs, dest, snapshot) + await dumpData(dest, consolidated.index, snapshot.storeHash, true) + + if (useCompiledPkgJson && pruned.files['package.json']) { + await fs.writeFile( + path.resolve(dest, 'package.json'), + await getDataRepository().readData(pruned.files['package.json'].hash) + ) + } else { + await fs.writeFile( + path.resolve(dest, 'package.json'), + JSON.stringify(pkg.data, undefined, 4), + ) + } + + if (!target?.skipPackage) { + const binaries = await buildBinaryDeps(dest, target) + if (target?.sign) { + for (const [k, v] of Object.entries(binaries)) { + await sign(v) + } + } + + if (pkg.data.engines?.node && !target?.snapshot && !target?.downloadOnly) { + await getNodeJsForPkg(dest, target) + } + } + + if (target?.buildLicense) { + getLogger().log('Building license') + const license = await buildLicense() + await getFs().writeFile(path.resolve(dest, 'LICENSE'), license) + } + + if (target?.external) { + const mapping = await installExternalPackages(dest, target.external, target) + const resolved = resolveBuildTarget(target) + const esbuildName = resolved.os === 'windows' ? `esbuild.exe` : 'esbuild' + const esbuildBinPath = path.resolve(dest, 'node_modules', 'esbuild', 'bin', esbuildName) + + async function maybeCopyEsbuildBinary() { + const dir = path.resolve(dest, 'node_modules', '@esbuild') + const files = await getFs().readDirectory(dir).catch(throwIfNotFileNotFoundError) + if (!files || files.length > 1 || files[0].type !== 'directory') return false + + // TODO: add `copyFile` to `system.ts` + const p = path.resolve(dir, files[0].name, 'bin', esbuildName) + const data = await getFs().readFile(p).catch(e => { + throwIfNotFileNotFoundError(e) + + return getFs().readFile(path.resolve(dir, files[0].name, esbuildName)) + }) + + await getFs().writeFile(esbuildBinPath, data) + + return true + } + + if (await maybeCopyEsbuildBinary()) { + if (target?.sign) { + // TODO: sign `esbuild` + } + + await getFs().deleteFile(path.resolve(dest, 'node_modules', '@esbuild')).catch(throwIfNotFileNotFoundError) + + if (target.snapshot) { + await getFs().writeFile(path.resolve(dest, 'tools', esbuildName), await getFs().readFile(esbuildBinPath)) + await getFs().deleteFile(path.resolve(dest, 'node_modules', 'esbuild')).catch(throwIfNotFileNotFoundError) + } else { + await patchEsbuildMain(path.resolve(dest, 'node_modules', 'esbuild'), esbuildBinPath) + + if (resolved.os === 'windows') { + await getFs().deleteFile(esbuildBinPath.replace('.exe', '')).catch(throwIfNotFileNotFoundError) + } + } + } + + // TODO: figure out why typescript can't find the lib files when not using SEA + // perhaps remove this if statement? + if (target.snapshot) { + // TODO: do we need to copy the other files? + const typescriptLibPath = path.resolve(dest, 'node_modules', 'typescript', 'lib') + const libFiles = await glob(getFs(), typescriptLibPath, ['lib.*.d.ts']) + // const minimalLibs = ['lib.es5.d.ts', 'lib.decorators.d.ts', 'lib.decorators.legacy.d.ts'] + for (const l of libFiles) { + const text = await getFs().readFile(path.resolve(typescriptLibPath, l), 'utf-8') + const stripped = text //stripComments() + await getFs().writeFile( + path.resolve(dest, 'dist', path.basename(l)), + stripped + ) + } + } + + // Remove installed deps from `package.json` + const copiedData = removeExternalDeps(target.external, pkg.data) + await fs.writeFile( + path.resolve(dest, 'package.json'), + JSON.stringify(copiedData, undefined, 4), + ) + + return { pruned, mapping } + } + + return { pruned } + + // TODO: write hash list + // TODO: sign everything +} + +function stripComments(text: string) { + const lines = text.split('\n') + let line = lines.length - 1 + for (; line > 0; line--) { + if (lines[line - 1].startsWith('/// <')) break + } + + const result = lines.slice(0, line) + const rest: string[] = [] + for (; line < lines.length; line++) { + const stripped = lines[line].replace(/\/\/.*/g, '') + if (stripped.trim()) { + rest.push(stripped) + } + } + + result.push(rest.join('\n').replace(/\/\*(\n|.)*\*\//g, '')) + + return result.join('\n') +} + +export async function createArchive(dir: string, dest: string, sign?: boolean) { + if (path.extname(dest) === '.tgz') { + const files = await glob(getFs(), dir, ['**/*', '**/.synapse']) + const tarball = createTarball(await Promise.all(files.map(async f => ({ + contents: Buffer.from(await getFs().readFile(f)), + mode: 0o755, + path: path.relative(dir, f), + })))) + + const zipped = await gzip(tarball) + await getFs().writeFile(dest, zipped) + } else if (path.extname(dest) === '.zip') { + try { + await createZipFromDir(dir, dest, true) + } catch (e) { + getLogger().log(`failed to use built-in zip command`, e) + const cwd = path.dirname(dir) + await runCommand('zip', ['-r', dest, path.basename(dir)], { cwd }) + } + } else { + throw new Error(`Not implemented: ${path.extname(dest)}`) + } + + // TODO: notarize for darwin +} + +function pruneObject(obj: Record, s: Set) { + for (const [k, v] of Object.entries(obj)) { + if (!s.has(k)) { + delete obj[k] + } + } +} + +function pruneSnapshot(snapshot: Snapshot, filesToKeep: string[], stripInternal = false) { + const s = new Set(filesToKeep) + if (snapshot.published) { + pruneObject(snapshot.published, s) + } + if (snapshot.pointers) { + pruneObject(snapshot.pointers, s) + } + + // Delete sourcemaps on types + const typesToKeep = new Set() + if (snapshot.moduleManifest) { + for (const [k, v] of Object.entries(snapshot.moduleManifest)) { + if (v.types) { + delete (v.types as any).sourcemap + typesToKeep.add(v.path.replace(/\.js$/, '.d.ts')) + } + } + } + + if (snapshot.types) { + for (const k of Object.keys(snapshot.types)) { + if (!typesToKeep.has(k)) { + delete snapshot.types[k] + } + } + } +} + +async function patchEsbuildMain(pkgDir: string, binPath: string) { + const main = path.resolve(pkgDir, 'lib', 'main.js') + + const mainText = await getFs().readFile(main, 'utf-8') + if (mainText.startsWith('var ESBUILD_BINARY_PATH = ')) { + // Maybe validate that the binary works? + return + } + + const relBinPath = path.relative(path.dirname(main), binPath) + + const patched = `var ESBUILD_BINARY_PATH = require("node:path").resolve(__dirname, "${relBinPath}");\n${mainText}` + await getFs().writeFile(main, patched) +} + +export async function installExternalPackages(pkgDir: string, external: string[], target?: Partial) { + const pkg = await getPackageJsonOrThrow(pkgDir) + const deps = pkg.data.dependencies + if (!deps) { + return + } + + const needsInstall: [string, string][] = [] + for (const [k, v] of Object.entries(deps)) { + if (external.find(s => k.startsWith(s))) { + needsInstall.push([k, v]) + } + } + + if (needsInstall.length === 0) { + return + } + + const { mapping } = await installModules(pkgDir, Object.fromEntries(needsInstall), target) + + return mapping +} + +async function verifyCodesign() { + // codesign --verify --deep --strict --verbose=2 +} + + +// Developer ID Application <-- use this cert +// https://appstoreconnect.apple.com/access/integrations/api + +async function codesign(fileName: string, certId: string, entitlementsPath?: string) { + const args = ['--sign', certId, '--timestamp', '--options', 'runtime', fileName] + if (entitlementsPath) { + args.unshift('--entitlements', entitlementsPath) + } + + await runCommand('codesign', args) +} + +interface ConnectCreds { + id: string + key: string // file path + issuer: string + teamId: string +} + +async function notarize(fileName: string, creds: ConnectCreds) { + const credsArgs = [ + '--key', creds.key, + '--key-id', creds.id, + '--issuer', creds.issuer, + '--team-id', creds.teamId + ] + + const args = [ + 'notarytool', + 'submit', fileName, + '--wait', + '--output-format', 'json', + ...credsArgs, + ] + + const res = JSON.parse(await runCommand('xcrun', args)) as { id: string; status: string; message: string } + if (res.status === 'Invalid') { + const logsRaw = await runCommand('xcrun', ['notarytool', 'log', res.id, ...credsArgs]) + const logs = JSON.parse(logsRaw) + + throw new Error(`Failed to notarize: ${logsRaw}`) + } +} + +const entitlements = ` + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-executable-page-protection + + com.apple.security.cs.allow-dyld-environment-variables + + com.apple.security.cs.disable-library-validation + + com.apple.security.get-task-allow + + + +` + +async function sign(fileName: string, entitlements?: string) { + if (process.platform !== 'darwin') { + return + } + + const keyId = process.env.SIGNING_KEY_ID + if (!keyId) { + throw new Error(`Missing environment variable: SIGNING_KEY_ID`) + } + + const entitlementsPath = entitlements ? path.resolve(path.dirname(fileName), 'tmp-entitlements.plist') : undefined + if (entitlements && entitlementsPath) { + await getFs().writeFile(entitlementsPath, entitlements) + } + + await codesign(fileName, keyId, entitlementsPath).catch(e => { + if ((e as any).stderr?.includes('is already signed')) { + return + } + }) + + if (entitlements && entitlementsPath) { + await getFs().deleteFile(entitlementsPath) + } +} + +export async function signWithDefaultEntitlements(fileName: string) { + return sign(fileName, entitlements) +} + +const thirdPartyNotice = ` +This file is based on or incorporates material from the projects listed below +(collectively "Third Party Code"). Cohesible is not the original author of the +Third Party Code. The original copyright notice and the license, under which +Cohesible received such Third Party Code, are set forth below. Such licenses and +notices are provided for informational purposes only. + +Cohesible, not the third party, licenses the Third Party Code to you under the +terms set forth at the start of this file. Cohesible reserves all other rights +not expressly granted under this agreement, whether by implication, estoppel +or otherwise. +`.trim() + +export async function buildLicense() { + // from 5.4.5 + const typescriptLicense = ` +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 + +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. + +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +`.trim() + + // from 0.20.2 + const esbuildLicense = ` +MIT License + +Copyright (c) 2020 Evan Wallace + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +`.trim() + + const nodeLicense = await github.downloadRepoFile('Cohesible', 'synapse-node-private', 'LICENSE') + + const terraformLicense = await github.downloadRepoFile('Cohesible', 'synapse-terraform-private', 'LICENSE') + + const synapseLicense = ` +Copyright (c) Cohesible, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 + +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. + +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +`.trim() + + // TODO: include TypeScript's third party notice + const sections = { + 'Node.js': nodeLicense.toString('utf-8'), + Terraform: terraformLicense.toString('utf-8'), + TypeScript: typescriptLicense, + esbuild: esbuildLicense, + } + + const license: string[] = [] + license.push(synapseLicense) + license.push('') + license.push(thirdPartyNotice) + license.push('') + + for (const [k, v] of Object.entries(sections)) { + const border = '-'.repeat(16) + license.push(`${border} ${k} ${border}`) + license.push('') + license.push(v) + license.push('') + } + + return license.join('\n') +} diff --git a/src/cli/commands.ts b/src/cli/commands.ts new file mode 100644 index 0000000..3258f84 --- /dev/null +++ b/src/cli/commands.ts @@ -0,0 +1,1515 @@ +import * as synapse from '..' +import * as path from 'node:path' +import { getObjectByPrefix } from '../build-fs/utils' +import { eagerlyStartDaemon, emitCommandEvent } from '../services/analytics' +import { getCiType, levenshteinDistance, memoize, toSnakeCase } from '../utils' +import { getWorkingDir, resolveProgramBuildTarget } from '../workspaces' +import { readKey, setKey } from './config' +import { RenderableError, colorize, printJson, printLine, stripAnsi } from './ui' +import { runWithContext, getBuildTargetOrThrow } from '../execution' +import { handleCompletion } from './completions/completion' +import { downloadNodeLib } from './buildInternal' +import { runInternalTestFile } from '../testing/internal' +import { getAuth } from '../auth' + + +interface TypeMap { + 'string': string + 'number': number + 'boolean': boolean +} + +export const enumTypeSym = Symbol('enumType') +export const fileTypeSym = Symbol('fileType') +export const unionTypeSym = Symbol('unionType') + +interface EnumType { + readonly [enumTypeSym]: Set +} + +interface FileType { + readonly [fileTypeSym]: Set // Valid suffixes +} + +interface UnionType { + readonly [unionTypeSym]: ArgType[] +} + +// No patterns = match all files +function createFileType(...values: string[]): FileType { + return { [fileTypeSym]: new Set(values) } +} + +const typescriptFileType = createFileType('.ts', '.tsx') + + +function createEnumType(...values: T): EnumType { + return { [enumTypeSym]: new Set(values) } +} + +function createUnionType(...types: T): UnionType> { + return { [unionTypeSym]: types } +} + +type FromArgType = T extends keyof TypeMap ? TypeMap[T] + : T extends (v: string) => Promise | infer U ? U + : T extends UnionType ? U + : T extends EnumType ? U + : T extends FileType ? string : never + +type ArgType = keyof TypeMap | ((v: string) => Promise | T) | EnumType | FileType | UnionType + +interface ArgumentOptions { + readonly description?: string + readonly allowMultiple?: boolean + readonly defaultValue?: FromArgType | (() => FromArgType) + readonly aliases?: string[] + readonly hidden?: boolean +} + +interface PositionalArgument extends ArgumentOptions { + readonly name: string + readonly type: T + readonly optional?: boolean + readonly minCount?: number +} + +interface OptionalArgument extends PositionalArgument { + readonly optional: true +} + +interface VarArgsArgument extends PositionalArgument { + readonly allowMultiple: true + readonly minCount?: number +} + +interface SwitchArgument extends ArgumentOptions { + readonly name: K + readonly type: T + readonly passthrough?: boolean + readonly shortName?: string + readonly environmentVariable?: string +} + +interface VarArgsOption extends SwitchArgument { + readonly allowMultiple: true +} + +interface PassthroughSwitch extends VarArgsOption<'string', 'targetArgs'> { + readonly passthrough: true +} + +// Special switch to allow `-- [arg1] [arg2] [arg3] ...` +const passthroughSwitch: PassthroughSwitch = { + name: 'targetArgs', + type: 'string', + allowMultiple: true, + passthrough: true, +} + +export interface RegisteredCommand { + readonly name: string + readonly fn: (...args: T) => Promise | void + readonly descriptor: CommandDescriptor +} + +type ExpandArgs = T extends [VarArgsArgument] + ? [...FromArgType[]] + : T extends [OptionalArgument, ...infer R] ? [FromArgType | undefined, ...ExpandArgs] + : T extends [PositionalArgument, ...infer R] ? [FromArgType, ...ExpandArgs] : [] + +type ExpandOptions = T extends [VarArgsOption, ...infer R] + ? { [P in K]+?: FromArgType[] } & ExpandOptions + : T extends [SwitchArgument, ...infer R] + ? { [P in K]+?: FromArgType } & ExpandOptions : {} + +interface CommandRequirements { + readonly program?: boolean + readonly process?: boolean + readonly project?: boolean +} + +export interface CommandDescriptor< + T extends PositionalArgument[] = PositionalArgument[], + U extends SwitchArgument[] = SwitchArgument[] +> { + // `internal` = exclude from public build + // `hidden` = include in public build but hide in UI + readonly hidden?: boolean + readonly internal?: boolean + readonly category?: string + readonly examples?: string[] + readonly aliases?: string[] + + // Maybe add `examplesWithCode` + // Which would be a short snippet of self-contained code + a command + // + // Example: + // export function main(...args: string[]) { + // console.log(`Hello, ${args[0]}!`) + // } + // + // synapse run -- world + // > Hello, world! + // + // + // Would only show with --help + + readonly description?: string + readonly helpDescription?: string // Longer description + readonly requirements?: CommandRequirements + readonly inferBuildTarget?: boolean + + readonly args?: T + readonly options?: U + + readonly isImportantCommand?: boolean +} + +const registeredCommands = new Map() +const aliasedCommands = new Map() + +export function registerCommand(name: string, fn: (...args: any[]) => Promise | void, descriptor: CommandDescriptor = {}) { + if (registeredCommands.has(name) || aliasedCommands.has(name)) { + throw new Error(`Command "${name}" has already been registered.`) + } + + if (descriptor.aliases) { + for (const n of descriptor.aliases) { + if (registeredCommands.has(n) || aliasedCommands.has(n)) { + throw new Error(`Command "${n}" has already been registered.`) + } + aliasedCommands.set(n, name) + } + } + + validateDescriptor(descriptor) + registeredCommands.set(name, { name, fn, descriptor }) +} + +export function registerTypedCommand< + const T extends PositionalArgument[] = PositionalArgument[], + const U extends SwitchArgument[] = SwitchArgument[] +>(name: string, descriptor: CommandDescriptor, fn: (...args: [...ExpandArgs, ExpandOptions]) => Promise | void) { + registerCommand(name, fn, descriptor) +} + +function unpackArgs(args: [...T, U]): [T, U] { + return [args.slice(0, -1) as any, args.at(-1) as any] +} + +interface ShowCommandsOptions { + includeInternal?: boolean + importantOnly?: boolean + indent?: number +} + +function showCommands(opt: ShowCommandsOptions = {}) { + const { + importantOnly = true, + includeInternal = false, + indent = 4, + } = opt + + function filter(desc: CommandDescriptor) { + if (importantOnly && !desc.isImportantCommand) { + return false + } + if (desc.internal && !includeInternal) { + return false + } + return !desc.hidden + } + + const filtered = [...registeredCommands].filter(([k, v]) => filter(v.descriptor)) + + const parts: [string, string][] = [] + for (const [k, v] of filtered) { + const suffix = v.descriptor.internal ? colorize('gray', ` [internal]`) : '' + const label = `${colorize('cyan', k)}${suffix}` + const desc = v.descriptor.description ? `${v.descriptor.description}` : '' + parts.push([label, desc]) + } + + if (parts.length === 0) { + return + } + + const stripped = parts.map(p => stripAnsi(p[0])) + const padding = stripped.sort((a, b) => a.length - b.length).at(-1)!.length + 2 + for (const [k, v] of parts) { + const label = k + ' '.repeat(padding - stripAnsi(k).length) + printLine(`${' '.repeat(indent)}${label}${v}`) + } +} + +export function showUsage() { + printLine('Usage: synapse [...options] [...arguments]') + printLine() + printLine('Commands:') + showCommands() +} + +// Really adhoc. I know there's way better ways to do fuzzy matching +function findPossibleSwaps(a: string, b: string) { + if (a.length > b.length) { + return findPossibleSwaps(b, a) + } + + let swaps = 0 + for (let i = 0; i < a.length; i++) { + if (a[i] === b[i - 1] && a[i - 1] === b[i]) { + swaps += 1 + } else if (a[i] === b[i + 1] && a[i + 1] === b[i]) { + swaps += 1 + } + } + + return swaps +} + +function didYouMean(cmd: string) { + const scores = [...registeredCommands.entries()] + .filter(x => !x[1].descriptor.internal && !x[1].descriptor.hidden) + .map(x => x[0]) + .map(c => [c, levenshteinDistance(c.slice(0, cmd.length), cmd)] as const) + .sort((a, b) => a[1] - b[1]) + + const bestScore = scores[0][1] + const invalidCmdMsg = `"${cmd}" is not a valid command.` + + // Take all commands with the best score + const matches = scores.filter(x => x[1] === bestScore).map(x => x[0]) + if (matches.length === 1) { + const suggestion = renderCmdSuggestion(matches[0], [], false) + printLine(`${invalidCmdMsg} Did you mean ${suggestion}`) + return + } + + // We'll break ties by checking possible tranpositions + const round2 = matches.map(x => [x, findPossibleSwaps(x, cmd)] as const).sort((a, b) => b[1] - a[1]) + const bestScore2 = round2[0][1] + const answers = round2.filter(x => x[1] === bestScore2).map(x => x[0]) + if (answers.length === 1) { + const suggestion = renderCmdSuggestion(answers[0], [], false) + printLine(`${invalidCmdMsg} Did you mean ${suggestion}`) + } else { + printLine(`${invalidCmdMsg} Did you mean:`) + const colWidth = 16 + const numCols = 4 + const numRows = Math.ceil(answers.length / numCols) + for (let i = 0; i < numRows; i++) { + let line = ' ' + for (let j = 0; j < numCols; j++) { + const m = answers[i * 4 + j] + if (!m) break + + const text = renderCmdSuggestion(m, [], false) + const width = stripAnsi(text).length + line += text + ' '.repeat(colWidth - width) + } + + printLine(line) + } + } +} + +export async function runWithAnalytics(name: string, cmd: () => Promise) { + eagerlyStartDaemon() + + let errorCode: string | undefined + const startTime = Date.now() + + try { + await cmd() + } catch (e) { + const code = (e as any).code + errorCode = code ? String(code) : (e as any).name + // TODO: extract traces that are from the CLI + // We could emit that data if we scrub filepaths + + throw e + } finally { + const duration = Date.now() - startTime + + try { + emitCommandEvent({ + name, + duration, + errorCode, + }) + } catch {} + } +} + +const oses = ['windows', 'linux', 'darwin'] +const archs = ['x64', 'aarch64'] + +const pairs: string[] = [] +for (const os of oses) { + for (const arch of archs) { + pairs.push(`${os}-${arch}`) + } +} + +const hostTargetType = createEnumType(pairs[0], ...pairs.slice(1)) + +const supportedIntegrations = ['local', 'aws', 'azure', 'gcp'] as const + +const varargsFiles = { + name: 'files', + type: typescriptFileType, + allowMultiple: true, +} satisfies PositionalArgument + +const objectHashType = (val: string) => getObjectByPrefix(val) + +const objectHashArg = { + name: 'objectHash', + type: objectHashType, +} satisfies PositionalArgument + +const deployTargetOption = { + name: 'target' as const, + shortName: 't', + type: createEnumType(...supportedIntegrations), + description: 'The default deployment target to use when synthesizing standard resources' +} satisfies SwitchArgument + +const buildTargetOptions = [ + { name: 'environment', type: 'string', environmentVariable: 'SYNAPSE_ENV', aliases: ['env'], hidden: true } +] as const satisfies SwitchArgument[] + +const compileOptions = [ + ...buildTargetOptions, + deployTargetOption, + { name: 'no-incremental', type: 'boolean', description: 'Disables incremental compilation' }, + { name: 'no-synth', type: 'boolean', description: 'Synthesis inputs are emitted instead of executed', hidden: true }, // TODO: better description + { name: 'no-infra', type: 'boolean', description: 'Disables generation of synthesis inputs', hidden: true }, + { name: 'exclude-providers', type: 'boolean', description: 'Removes generated provider *.d.ts files from program analysis', hidden: true }, + { name: 'skip-install', type: 'boolean' }, + { name: 'host-target', type: hostTargetType, hidden: true }, + { name: 'strip-internal', type: 'boolean', hidden: true } +] as const satisfies SwitchArgument[] + +const deployOptions = [ + ...buildTargetOptions, + deployTargetOption, + // This will behave like `plan` + { + name: 'dry-run', + type: 'boolean', + description: 'Shows the predicted changes to a deployment without applying them.' + }, + // Hidden because not tested + { name: 'refresh', type: 'boolean', description: 'Fetches the state of remote resources instead of using the saved state', hidden: true }, + { name: 'terraform-path', type: 'string', hidden: true }, + { name: 'provider-server-port', type: 'number', description: 'Use a specific port for the Synapse resource server', hidden: true }, + { name: 'sync-after', type: 'boolean', hidden: true }, + { name: 'symbol', type: 'string', hidden: true, allowMultiple: true }, + { name: 'use-optimizer', type: 'boolean', hidden: true }, +] as const satisfies SwitchArgument[] + +registerTypedCommand( + 'compile', + { + args: [varargsFiles], + options: [...compileOptions, { name: 'force-infra', type: 'string', allowMultiple: true, hidden: true }], + requirements: { program: true }, + inferBuildTarget: true, + description: 'Converts program source code into deployable artifacts' + }, + async (...args) => { + const [files, opt] = unpackArgs(args) + + await synapse.compile(files, { + deployTarget: opt['target'], + noInfra: opt['no-infra'], + noSynth: opt['no-synth'], + incremental: !opt['no-incremental'], + excludeProviderTypes: opt['exclude-providers'], + skipInstall: opt['skip-install'], + hostTarget: opt['host-target'], + forcedInfra: opt['force-infra'], + stripInternal: opt['strip-internal'], + }) + } +) + +registerTypedCommand( + 'watch', + { + internal: true, + options: [{ name: 'auto-deploy', type: 'boolean' }], + }, + async (...args) => { + const [files, opt] = unpackArgs(args) + + await synapse.watch(undefined, { + autoDeploy: opt['auto-deploy'], + }) + }, +) + +registerTypedCommand( + 'bundle-to-sea', + { + internal: true, + args: [{ name: 'dir', type: 'string' }], + }, + async (...args) => { + await synapse.convertBundleToSea(args[0]) + } +) + +registerTypedCommand( + 'deploy', + { + isImportantCommand: true, + args: [varargsFiles], + options: [...deployOptions, { name: 'rollback-if-failed', type: 'boolean', hidden: true }, { name: 'plan-depth', type: 'number', hidden: true }], + requirements: { program: true, process: true }, + inferBuildTarget: true, + description: 'Creates or updates a deployment' + }, + async (...args) => { + const [files, opt] = unpackArgs(args) + + // XXX: hardcoded + if (opt.target === 'azure' || opt.target === 'gcp') { + throw new Error(`The cloud target "${opt.target}" is not yet implemented.`) + } + + if (opt['dry-run']) { + return synapse.plan(files, { + symbols: opt['symbol'], + forceRefresh: opt['refresh'], + planDepth: opt['plan-depth'], + }) + } + + await synapse.deploy(files, { + symbols: opt['symbol'], + forceRefresh: opt['refresh'], + deployTarget: opt['target'], + syncAfter: opt['sync-after'], + rollbackIfFailed: opt['rollback-if-failed'], + useOptimizer: opt['use-optimizer'], + }) + } +) + +registerTypedCommand( + 'pull', + { + internal: true, + options: [{ name: 'fail-if-empty', type: 'boolean' }], + }, + async (...args) => { + const [_, opt] = unpackArgs(args) + + const bt = getBuildTargetOrThrow() + if (bt.deploymentId) { + return synapse.syncModule(bt.deploymentId) + } + + if (opt['fail-if-empty']) { + throw new Error(`Nothing to pull`) + } + printLine('Nothing to pull') + } +) + +registerTypedCommand( + 'destroy', + { + isImportantCommand: true, + args: [varargsFiles], + options: [...deployOptions, { name: 'deploymentId', type: 'string', hidden: true }, { name: 'tests-only', type: 'boolean', hidden: true }], + requirements: { program: true, process: true }, + inferBuildTarget: true, + description: 'Deletes resources in a deployment' + }, + async (...args) => { + const [files, opt] = unpackArgs(args) + + await synapse.destroy(files, { + dryRun: opt['dry-run'], + symbols: opt['symbol'], + forceRefresh: opt['refresh'], + deploymentId: opt.deploymentId, + useTests: opt['tests-only'], + }) + } +) + +registerTypedCommand( + 'rollback', + { + internal: true, + options: [ + ...buildTargetOptions, + { name: 'sync-after', type: 'boolean', hidden: true }, + ], + }, + async (...args) => { + const [files, opt] = unpackArgs(args) + + await synapse.rollback('', opt as any) + } +) + +registerTypedCommand( + 'test', + { + isImportantCommand: true, + args: [varargsFiles], + options: [ + ...buildTargetOptions, + { name: 'destroy-after', type: 'boolean', hidden: true }, + { name: 'rollback-if-failed', type: 'boolean', hidden: true } + ], + requirements: { program: true, process: true }, + inferBuildTarget: true, + description: 'Deploys and runs test resources' + }, + async (...args) => { + const [files, opt] = unpackArgs(args) + + await synapse.runTests(files, { + destroyAfter: opt['destroy-after'], + rollbackIfFailed: opt['rollback-if-failed'], + }) + } +) + +registerTypedCommand( + 'print-types', + { + internal: true, + args: [], + options: [ + + ], + }, + (opt) => synapse.printTypes() +) + +registerTypedCommand( + 'show-object', + { + internal: true, + args: [objectHashArg], + options: [ + { name: 'captured', type: 'boolean' } + ], + }, + (obj, opt) => synapse.showRemoteArtifact(obj, opt) +) + +registerTypedCommand( + 'publish', + { + hidden: true, + options: [ + { name: 'local', type: 'boolean' }, + { name: 'global', type: 'boolean', hidden: true }, + { name: 'dry-run', type: 'boolean', hidden: true }, + { name: 'skip-install', type: 'boolean', hidden: true }, + { name: 'archive', type: 'string', hidden: true }, + { name: 'new-format', type: 'boolean', hidden: true }, + ...buildTargetOptions, + ], + }, + async (opt) => { + await synapse.publish('', { + ...opt, + dryRun: opt['dry-run'], + skipInstall: opt['skip-install'], + archive: opt['archive'], + newFormat: opt['new-format'], + environmentName: opt['environment'], + }) + } +) + +registerTypedCommand( + 'gc', + { + internal: true, + options: [ + { name: 'dry-run', type: 'boolean' } + ], + }, + async (opt) => { + await synapse.collectGarbage('', { + ...opt, + dryRun: opt['dry-run'], + }) + } +) + +registerTypedCommand( + 'gc-resources', + { + internal: true, + options: [ + { name: 'dry-run', type: 'boolean' } + ], + }, + async (opt) => { + await synapse.collectGarbageResources('', { + ...opt, + dryRun: opt['dry-run'], + }) + } +) + + +registerTypedCommand( + 'show', + { + internal: true, // Temporary + args: [{ name: 'symbols', type: 'string', allowMultiple: true }], + options: buildTargetOptions + }, + async (...args) => { + const [symbols, opt] = unpackArgs(args) + + await synapse.show(symbols, ) + } +) + +// `synapse help` should only show the most important commands/options rather than +// dumping everything and having the user dig through a bunch of text. The `--all` +// switch can be used if someone really wants the wall of text. +registerTypedCommand( + 'help', + { + isImportantCommand: true, + args: [], + description: 'Shows additional information', + options: [{ name: 'all', type: 'boolean', description: 'Shows everything', hidden: true }], + }, + async (...args) => { + printLine(`The built-in \`help\` command isn't done yet.`) + printLine(`So here's a link instead: https://github.com/Cohesible/synapse/blob/main/docs/getting-started.md`) + } +) + +registerTypedCommand( + 'migrate', + { + internal: true, + args: [varargsFiles], + options: [{ name: 'reset', type: 'boolean' }, { name: 'dryRun', type: 'boolean' }, ...buildTargetOptions], + }, + async (...args) => { + const [files, opt] = unpackArgs(args) + + await synapse.migrateIdentifiers(files, opt) + } +) + + +registerTypedCommand( + 'status', + { + internal: true, + options: [{ name: 'verbose', shortName: 'v', type: 'boolean' }] + }, + opt => synapse.showStatus(opt) +) + +registerTypedCommand( + 'run', + { + isImportantCommand: true, + args: [{ name: 'target', type: createUnionType(typescriptFileType, 'string'), optional: true }], + options: [passthroughSwitch, { name: 'skipValidation', type: 'boolean', hidden: true }, { name: 'skipCompile', type: 'boolean', hidden: true }, ...buildTargetOptions], + inferBuildTarget: true, + description: 'Executes a target file/script. Uses an executable in the current application by default.', + }, + async (target, opt) => { + await synapse.run(target, opt.targetArgs ?? [], opt) + } +) + +registerTypedCommand( + 'run-internal-test', + { + internal: true, + args: [{ name: 'file', type: typescriptFileType }], + }, + async (target) => { + await runInternalTestFile(path.resolve(getWorkingDir(), target)) + } +) + +registerTypedCommand( + 'export-identity', + { + internal: true, + args: [{ name: 'destination', type: 'string' }], + }, + async (target) => { + const auth = getAuth() + await auth.exportIdentity(path.resolve(getWorkingDir(), target)) + } +) + +registerTypedCommand( + 'import-identity', + { + internal: true, + args: [{ name: 'file', type: 'string' }], + requirements: { program: false } + }, + async (target) => { + const auth = getAuth() + await auth.importIdentity(!target ? '-' : path.resolve(getWorkingDir(), target)) + } +) + +registerTypedCommand( + 'add', + { + internal: true, + args: [{ name: 'packages', type: 'string', allowMultiple: true, min: 1 }], + options: [ + { name: 'dev', shortName: 'd', type: 'boolean' }, + { name: 'mode', type: createEnumType('all', 'types'), hidden: true } + ] + }, + async (...args) => { + const [packages, opt] = unpackArgs(args) + await synapse.install(packages, opt) + } +) + +registerTypedCommand( + 'remove', + { + internal: true, + args: [{ name: 'packages', type: 'string', allowMultiple: true, min: 1 }], + options: [] + }, + async (...args) => { + const [packages, opt] = unpackArgs(args) + await synapse.install(packages, { ...opt, remove: true }) + } +) + +/** @deprecated */ +registerTypedCommand( + 'clear-cache', + { + internal: true, + args: [{ name: 'caches', type: 'string', allowMultiple: true }] + }, + async (...args) => { + const [caches, opt] = unpackArgs(args) + await synapse.clearCache(caches[0], opt) + } +) + +registerTypedCommand( + 'dump-fs', + { + internal: true, + args: [{ name: 'fs', optional: true, type: createUnionType(createEnumType('program', 'process', 'package'), objectHashType) }], + options: [...buildTargetOptions, { name: 'block', type: 'boolean', hidden: true }, { name: 'debug', type: 'boolean', hidden: true, defaultValue: true }] + }, + async (fs, opt) => { + await synapse.emitBfs(fs, opt) + } +) + +registerTypedCommand( + 'emit', + { + args: [], + options: [...buildTargetOptions, { name: 'outDir', type: 'string' }, { name: 'block', type: 'boolean', hidden: true }, { name: 'debug', type: 'boolean', hidden: true }, { name: 'no-optimize', type: 'boolean', hidden: true }] + }, + async (opt) => { + await synapse.emitBfs('package', { ...opt, isEmit: true }) + } +) + + +registerTypedCommand( + 'show-logs', + { + hidden: true, + options: [{ name: 'list', type: 'boolean' }] + }, + async (...args) => { + const [_, opt] = unpackArgs(args) + await synapse.showLogs(opt.list ? 'list' : '') + } +) + +registerTypedCommand( + 'login', + { + internal: true, + options: [{ name: 'machine', type: 'boolean', hidden: true }], + requirements: { program: false } + }, + async (...args) => { + const [_, opt] = unpackArgs(args) + if (opt.machine) { + return synapse.machineLogin() + } + await synapse.login() + } +) + +registerTypedCommand( + 'clean', + { + args: [], + options: [{ name: 'packages', type: 'boolean', description: 'Clears the packages cache' }], + requirements: { program: true }, + }, + (opt) => synapse.clean(opt), +) + +registerTypedCommand( + 'install', + { + hidden: true, + args: [], + requirements: { program: true }, + }, + (opt) => synapse.install([]), +) + +registerTypedCommand( + 'locked-install', + { + internal: true, + args: [], + requirements: { program: true }, + }, + (opt) => synapse.lockedInstall(), +) + + +registerTypedCommand( + 'list-install', + { + internal: true, + args: [], + requirements: { program: true }, + }, + (opt) => synapse.listInstallTree(), +) + +registerTypedCommand( + 'build', + { + hidden: true, + args: [], + }, + (p) => synapse.buildExecutables(p) +) + +registerTypedCommand( + 'diff-objects', + { + internal: true, + args: [{ name: 'obj1', type: objectHashType }, { name: 'obj2', type: objectHashType }], + }, + (a, b) => synapse.diffObjectsCmd(a, b) +) + +registerTypedCommand( + 'diff-indices', + { + internal: true, + args: [{ name: 'obj1', type: objectHashType }, { name: 'obj2', type: objectHashType }], + }, + (a, b) => synapse.diffIndicesCmd(a, b) +) + +registerTypedCommand( + 'diff-file', + { + internal: true, + args: [{ name: 'file', type: 'string' }], + options: [{ name: 'commitsBack', type: 'number' }], + }, + (a, opt) => synapse.diffFileCmd(a, opt) +) + +registerTypedCommand( + 'repl', + { + internal: true, // Temporary + args: [{ name: 'file', type: typescriptFileType }], + options: buildTargetOptions, + }, + (a, opt) => synapse.replCommand(a) +) + +registerTypedCommand( + 'test-glob', + { + internal: true, + args: [{ name: 'patterns', type: 'string', allowMultiple: true }], + }, + async (...args) => { + const [patterns, opt] = unpackArgs(args) + await synapse.testGlob(patterns, opt) + } +) + +registerTypedCommand( + 'load-block', + { + internal: true, + args: [{ name: 'path', type: 'string' }, { name: 'destination', type: 'string', optional: true }], + options: buildTargetOptions, + }, + async (...args) => { + await synapse.loadBlock(args[0], args[1]) + } +) + +registerTypedCommand( + 'dump-state', + { + internal: true, + args: [{ name: 'path', type: 'string', optional: true }], + options: buildTargetOptions, + }, + async (...args) => { + await synapse.dumpState(args[0]) + } +) + +registerTypedCommand( + 'load-state', + { + internal: true, + args: [{ name: 'path', type: 'string' }], + options: buildTargetOptions, + }, + async (...args) => { + await synapse.loadState(args[0]) + } +) + +registerTypedCommand( + 'bundle', + { + internal: true, + args: [{ name: 'target', type: hostTargetType, optional: true }], + options: [ + { name: 'snapshot', type: 'boolean' }, + { name: 'snapshotOnly', type: 'boolean' }, + { name: 'sea', type: 'boolean' }, + { name: 'production', type: 'boolean' }, + { name: 'lto', type: 'boolean' }, + { name: 'integrationsOnly', type: 'boolean' }, + { name: 'seaOnly', type: 'boolean' }, + { name: 'stagingDir', type: 'string' }, + { name: 'downloadOnly', type: 'boolean' }, + { name: 'preserveSource', type: 'boolean' }, + { name: 'libc', type: 'string' }, + { name: 'integration', type: 'string', allowMultiple: true }, + { name: 'seaPrep', type: 'boolean' } + ] + }, + (target, opt) => synapse.internalBundle(target, opt) +) + +registerTypedCommand( + 'download-node-lib', + { + internal: true, + args: [], + }, + async (...args) => { + await downloadNodeLib('Cohesible', 'synapse-node-private') + } +) + +registerTypedCommand( + 'test-zip', + { + internal: true, + args: [{ name: 'dir', type: 'string' }, { name: 'dest', type: 'string', optional: true }], + }, + async (...args) => { + await synapse.testZip(args[0], args[1]) + } +) + + +registerTypedCommand( + 'convert-primordials', + { internal: true, args: [{ name: 'files', type: 'string', allowMultiple: true }] }, + (...args) => synapse.convertPrimordials(args.slice(0, -1) as string[]), +) + + +registerTypedCommand( + 'backup', + { internal: true, args: [{ name: 'destination', type: 'string' }] }, + (...args) => synapse.backup(args[0]), +) + +registerTypedCommand( + 'explain', + { internal: true, args: [{ name: 'symbol', type: 'string' }] }, + (...args) => synapse.explain(args[0]), +) + + +registerTypedCommand( + 'list-processes', + { internal: true }, + (...args) => synapse.listProcesses(), +) + +registerTypedCommand( + 'test-gc-daemon', + { internal: true }, + (...args) => synapse.testGcDaemon(), +) + +registerTypedCommand( + 'inspect-block', + { + internal: true, + args: [{ name: 'target', type: 'string'}], + }, + (target, opt) => synapse.inspectBlock(target, opt) +) + + +registerTypedCommand( + 'commands', + { + internal: true, + options: [{ name: 'internal', type: 'boolean' }] + }, + (opt) => showCommands({ includeInternal: opt.internal }) +) + +registerTypedCommand( + 'quote', + { + isImportantCommand: true, + description: 'Prints a motivational quote queried from a public Synapse application', + }, + () => synapse.quote() +) + +registerTypedCommand( + 'config', + { + hidden: true, + description: 'Get or set a key in the user config file', + // TODO: need to make get/set explicit. This is ok for now though + args: [{ name: 'key', type: 'string' }, { name: 'value', type: JSON.parse, optional: true }], + }, + async (...args) => { + if ((args as any).length === 2) { + const val = await readKey(args[0]) + printJson(val) + } else { + await setKey(args[0], args[1]) + } + } +) + +const getAllCommands = memoize(() => Object.fromEntries([...registeredCommands].map(([k, v]) => [k, v.descriptor]))) + +registerTypedCommand( + 'completion', + { options: [passthroughSwitch], hidden: true, requirements: { program: false } }, + (opt) => handleCompletion(opt.targetArgs ?? [], getAllCommands()), +) + +registerCommand('test-find-local', () => synapse.findLocalResources([]), { internal: true }) + +registerTypedCommand( + 'deploy-modules', + { + internal: true, + args: [{ name: 'files', type: typescriptFileType, allowMultiple: true }] + }, + async (...args) => { + const [files] = unpackArgs(args) + await synapse.deployModules(files) + } +) + +registerTypedCommand( + 'taint', + { + internal: true, + args: [{ name: 'resourceId', type: 'string' }], + }, + (a, opt) => synapse.taint(a, opt) +) + +registerTypedCommand( + 'delete-resource', + { + internal: true, + args: [{ name: 'resourceId', type: 'string' }], + options: [{ name: 'force', type: 'boolean' }, ...buildTargetOptions] + }, + (a, opt) => synapse.deleteResource(a, opt) +) + +registerTypedCommand( + 'list-commits', + { + internal: true, + requirements: { process: true }, + options: [{ name: 'useProgram', type: 'boolean' }] + }, + async (opt) => await synapse.listCommitsCmd('', opt), +) + +registerTypedCommand( + 'init', + { + // Only important for new users + // isImportantCommand: true, + description: 'Creates a new package in the current directory', + options: [ + { name: 'template', type: createEnumType('hello-world', 'react') } + ] + }, + (opt) => synapse.init(opt) +) + +export function isEnumType(type: ArgType): type is EnumType { + return typeof type === 'object' && !!(type as any)[enumTypeSym] +} + +function isUnionType(type: ArgType): type is UnionType { + return typeof type === 'object' && !!(type as any)[unionTypeSym] +} + +export function isFileType(type: ArgType): type is FileType { + return typeof type === 'object' && !!(type as any)[fileTypeSym] +} + +function parseNumber(arg: string) { + const n = Number(arg) + if (isNaN(n)) { + throw new Error('Not a number') + } + return n +} + +async function parseArg(val: string, type: ArgType) { + if (type === 'number') { + return parseNumber(val) + } + + if (typeof type === 'function') { + return type(val) + } + + if (isFileType(type)) { + const extnames = type[fileTypeSym] + const ext = path.extname(val) // TODO: support extnames with multiple dots? + if (!extnames.has(ext)) { + throw new Error(`Invalid value "${val}". Expected a filename ending in one of: ${[...extnames].join(', ')}`) + } + + return val + } + + if (isUnionType(type)) { + const types = type[unionTypeSym] + const errors: [ArgType, Error][] = [] + for (const t of types) { + try { + return await parseArg(val, t) + } catch(e) { + errors.push([t, e as Error]) + } + } + + throw new AggregateError(errors) + } + + if (isEnumType(type)) { + const s = type[enumTypeSym] + if (!s.has(val)) { + throw new Error(`Invalid value "${val}". Expected one of: ${[...s].join(', ')}`) + } + + return val + } + + return val +} + +function showHelp(desc: CommandDescriptor) { + // If command has passthrough switch we need to treat it differently +} + +function validateDescriptor(desc: CommandDescriptor) { + // Argument rules: + // 1. Commands with optional args cannot have a rest argument + // 2. Rest args must be the last argument + // 3. Optional args cannot precede required args + // 4. Maximum of 1 rest arg + + if (!desc.args) { + return + } + + const restIndex = desc.args.findIndex(x => x.allowMultiple) + if (restIndex !== -1 && restIndex !== desc.args.length - 1) { + throw new Error(`Rest arg must come at the end`) + } + + if (desc.args.filter(x => x.allowMultiple).length > 1) { + throw new Error(`Multiple rest args are not allowed`) + } + + const optionalArgs = desc.args.filter(x => x.optional) + const hasOptional = optionalArgs.length > 0 + if (hasOptional) { + if (restIndex !== -1) { + throw new Error(`Cannot combine rest arg with optional args`) + } + + const firstOptional = optionalArgs[0] + const rem = desc.args.slice(desc.args.indexOf(firstOptional) + 1) + const required = rem.filter(x => !x.optional) + if (required.length > 0) { + throw new Error(`Required args cannot come before optional args`) + } + } +} + +async function parseArgs(args: string[], desc: CommandDescriptor) { + let argPosition = 0 + let invalidPositionalArgs = 0 + + const parsedArgs: any[] = [] + const options: Record = {} + + const errors: [string, Error][] = [] + const unknownOptions: string[] = [] + + for (let i = 0; i < args.length; i++) { + const a = args[i] + if (a === '--') { + // If the command doesn't support passthrough args, it's possibly a typo + const passthroughOpt = desc.options?.find(x => x.passthrough) + if (!passthroughOpt) { + errors.push([a, new Error(`Passthrough arguments not supported`)]) + continue + } + + // Consume the remaining args + options[passthroughOpt.name] = args.slice(i + 1) + break + } + + const isLongSwitch = a.startsWith('--') + const isShortSwitch = !isLongSwitch && a.startsWith('-') + if (isShortSwitch || isLongSwitch) { + const n = a.slice(isShortSwitch ? 1 : 2) + const opt = isLongSwitch + ? desc.options?.find(x => x.name === n || x.aliases?.includes(n)) + : desc.options?.find(x => x.shortName === n) + + if (!opt) { + unknownOptions.push(n) + continue + } + + if (opt.type === 'boolean') { + options[opt.name] = true + continue + } + + if (i === args.length - 1) { + errors.push([a, new Error(`Missing value`)]) + break + } + + const arg = args[++i] + if (arg.startsWith('-')) { + // User probably forgot to add a value? + } + + try { + const parsed = await parseArg(arg, opt.type) + if (opt.allowMultiple) { + const arr = options[opt.name] ??= [] + arr.push(parsed) + } else if (opt.name in options) { + errors.push([a, new Error('Duplicate option')]) + } else { + options[opt.name] = parsed + } + } catch (e) { + errors.push([a, e as any]) + } + } else { + const currentArg = desc.args?.[argPosition] + if (!currentArg) { + errors.push([a, new Error('Unknown argument')]) + continue + } + + try { + const parsed = await parseArg(a, currentArg.type) + if (!currentArg.allowMultiple) { + argPosition += 1 + } + + parsedArgs.push(parsed) + } catch (e) { + invalidPositionalArgs += 1 + errors.push([a, e as any]) + } + } + } + + const allowMultipleArg = desc.args?.find(a => a.allowMultiple) + const minArgs = (desc.args?.filter(x => !x.allowMultiple && !x.optional).length ?? 0) + (allowMultipleArg?.minCount ?? 0) + const providedArgs = parsedArgs.length + invalidPositionalArgs + if (providedArgs < minArgs) { + for (let i = providedArgs; i < parsedArgs.length; i++) { + const a = desc.args![i] + if (a.allowMultiple) break + + errors.push([a.name, new Error('Missing argument')]) + } + + if (allowMultipleArg?.minCount) { + errors.push([ + allowMultipleArg.name, + new Error(`Requires at least ${allowMultipleArg.minCount} argument${allowMultipleArg.minCount > 1 ? 's' : ''}`) + ]) + } + } + + if (errors.length > 0) { + throw new RenderableError('Invalid arguments', () => { + for (const [n, e] of errors) { + printLine(colorize('brightRed', `${n} - ${e.message}`)) + } + }) + } + + const argCountWithOptional = desc.args?.filter(x => !x.allowMultiple).length ?? 0 + + // Fill with default/`undefined` + while (parsedArgs.length < argCountWithOptional) { + const arg = desc.args?.[parsedArgs.length] + const defaultValue = arg?.defaultValue + if (typeof defaultValue === 'function') { + parsedArgs.push(await defaultValue()) + } else { + parsedArgs.push(defaultValue) + } + } + + return { args: parsedArgs, options } +} + +async function getBuildTarget(cmd: CommandDescriptor, params: string[]) { + const cwd = process.cwd() + const environmentIndex = params.indexOf('--environment') + const environmentName = (environmentIndex !== -1 ? params[environmentIndex+1] : undefined) ?? process.env['SYNAPSE_ENV'] + const res = await resolveProgramBuildTarget(cwd, { environmentName }) + if (!res && cmd.inferBuildTarget) { + const programFiles = params.filter(x => x.match(/\.tsx?$/)) + if (programFiles.length > 1) { + throw new RenderableError('Failed to infer build target', () => { + printLine(colorize('brightRed', 'Package-less builds with multiple files are not supported')) + printLine() + printLine('Create a `package.json` file in the root of your project first.') + printLine('The file can contain an empty object: {}') + // TODO: add link to docs on `package.json` + + }) + } + + return resolveProgramBuildTarget(cwd, { program: programFiles[0], environmentName }) + } + + return res +} + +function getCommand(cmd: string) { + const name = aliasedCommands.get(cmd) ?? cmd + + return registeredCommands.get(name) +} + + +export async function executeCommand(cmd: string, params: string[]) { + const command = getCommand(cmd) + if (!command) { + throw new RenderableError(`Invalid command: ${cmd}`, () => didYouMean(cmd)) + } + + if (command.descriptor.requirements?.program === false) { + const parsed = await synapse.runTask('parse', cmd, () => parseArgs(params, command.descriptor), 1) + const args = [...parsed.args, parsed.options] + + return synapse.runTask('run', cmd, () => command.fn(...args), 1) + } + + const buildTarget = await getBuildTarget(command.descriptor, params) // 3ms or so + if (!buildTarget) { + if (command.descriptor.requirements) { + throw new RenderableError('No build target', () => { + printLine(colorize('brightRed', 'No build target found')) + }) + } + synapse.getLogger().debug(`No build target found`) + } else { + synapse.getLogger().debug(`Using resolved build target`, buildTarget) + } + + await runWithContext({ buildTarget }, async () => { + const parsed = await synapse.runTask('parse', cmd, () => parseArgs(params, command.descriptor), 1) + const args = [...parsed.args, parsed.options] + await synapse.runTask('run', cmd, () => command.fn(...args), 1) + }) +} + +function _inferCmdName() { + const execName = path.basename(process.env['SYNAPSE_PATH'] || process.execPath) + if (execName === 'node' || execName === 'node.exe') { + return 'synapse' + } + + const extLength = path.extname(execName).length + return extLength ? execName.slice(0, -extLength) : execName +} + +const inferCmdName = memoize(_inferCmdName) + +export function renderCmdSuggestion(commandName: string, args: string[] = [], includeExec = true) { + const parts = includeExec ? [inferCmdName(), commandName, ...args] : [commandName, ...args] + + return colorize('cyan', parts.join(' ')) +} + +export function removeInternalCommands() { + for (const [k, v] of registeredCommands) { + if (v.descriptor.internal) { + registeredCommands.delete(k) + } + } +} diff --git a/src/cli/completions/completion.sh b/src/cli/completions/completion.sh new file mode 100755 index 0000000..839f6a5 --- /dev/null +++ b/src/cli/completions/completion.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +if type complete &>/dev/null; then + _synapse_completion () { + local words cword + if type _get_comp_words_by_ref &>/dev/null; then + _get_comp_words_by_ref -n = -n @ -n : -w words -i cword + else + cword="$COMP_CWORD" + words=("${COMP_WORDS[@]}") + fi + + local si="$IFS" + if ! IFS=$'\n' COMPREPLY=($(COMP_CWORD="$cword" \ + COMP_LINE="$COMP_LINE" \ + COMP_POINT="$COMP_POINT" \ + synapse completion -- "${words[@]}" \ + 2>/dev/null)); then + local ret=$? + IFS="$si" + return $ret + fi + IFS="$si" + if type __ltrim_colon_completions &>/dev/null; then + __ltrim_colon_completions "${words[cword]}" + fi + } + complete -o default -F _synapse_completion synapse +elif type compdef &>/dev/null; then # zsh + _synapse_completion() { + compadd -- $(COMP_CWORD=$((CURRENT-1)) \ + COMP_LINE=$BUFFER \ + COMP_POINT=0 \ + synapse completion -- "${words[@]}" \ + 2>/dev/null) + } + compdef _synapse_completion synapse +elif type compctl &>/dev/null; then + _synapse_completion () { + local cword line point words si + read -Ac words + read -cn cword + let cword-=1 + read -l line + read -ln point + si="$IFS" + if ! IFS=$'\n' reply=($(COMP_CWORD="$cword" \ + COMP_LINE="$line" \ + COMP_POINT="$point" \ + synapse completion -- "${words[@]}" \ + 2>/dev/null)); then + + local ret=$? + IFS="$si" + return $ret + fi + IFS="$si" + } + compctl -K _synapse_completion synapse +fi diff --git a/src/cli/completions/completion.ts b/src/cli/completions/completion.ts new file mode 100644 index 0000000..5d207a2 --- /dev/null +++ b/src/cli/completions/completion.ts @@ -0,0 +1,177 @@ +import { getLogger } from '../..' +import * as path from 'node:path' +import { getFs } from '../../execution' +import { glob } from '../../utils/glob' +import { CommandDescriptor, enumTypeSym, fileTypeSym, isEnumType, isFileType } from '../commands' +// import { createFileAsset } from 'synapse:lib' + +// const completionsScript = createFileAsset(path.resolve(__dirname, 'completion.sh')) +// completionsScript. + +function parseArgsForCompletion(args: string[], desc: CommandDescriptor) { + let argPosition = 0 + + for (let i = 0; i < args.length; i++) { + const a = args[i] + if (a === '--' && i < args.length - 1) { + return false + } + + const isLongSwitch = a.startsWith('--') + const isShortSwitch = !isLongSwitch && a.startsWith('-') + if (isShortSwitch || isLongSwitch) { + const n = a.slice(isShortSwitch ? 1 : 2) + const opt = isLongSwitch + ? desc.options?.find(x => x.name === n || x.aliases?.includes(n)) + : desc.options?.find(x => x.shortName === n) + + if (!opt || opt.type === 'boolean') { + continue + } + + if (i === args.length - 1 || i === args.length - 2) { + return { ...opt, kind: 'option' } as const + } + } else { + const currentArg = desc.args?.[argPosition] + if (!currentArg) { + return + } + + if (!currentArg.allowMultiple) { + if (i === args.length - 1) { + break + } + argPosition += 1 + } + } + } + + const arg = desc.args?.[argPosition] + + return arg ? { ...arg, kind: 'arg' } as const : undefined +} + +const env = process.env.SYNAPSE_ENV +const showInternal = env !== 'production' + +export async function handleCompletion(args: string[], commands: Record) { + const { COMP_CWORD, COMP_LINE, COMP_POINT, COMP_FISH } = process.env + + if (COMP_CWORD === undefined || COMP_LINE === undefined || COMP_POINT === undefined) { + return + } + + const w = +COMP_CWORD + const words = args.map(unescape) + const word = words[w] ?? '' + + // Find a command + if (w === 1) { + const matches = Object.keys(commands) + .filter(x => !commands[x].hidden && (showInternal || !commands[x].internal)) + .filter(x => x.startsWith(word)) + + // Try looking for a file to execute + // This is 100% not the correct/best way to do completions for files + if (matches.length === 0) { + const cwd = process.cwd() + const r = path.resolve(cwd, word) + const dir = r !== cwd ? word.endsWith('/') ? word : path.dirname(r) : cwd + const base = r !== cwd && !word.endsWith('/') ? path.basename(r) : '' + const files = await getFs().readDirectory(dir) + + const lastSep = word.lastIndexOf('/') + const prefix = lastSep !== -1 ? word.slice(0, lastSep + 1) : '' + + const prefixMatched = files + .filter(x => x.name.startsWith(base)) + .filter(x => x.type === 'directory' || ((x.name.endsWith('.ts') && !x.name.endsWith('.d.ts'))|| x.name.endsWith('.js'))) + .map(x => `${prefix}${x.name}`) + + return emitCompletions(prefixMatched) + } + + return emitCompletions(matches) + } + + const cmd = commands[words[1]] + if (!cmd) { + return + } + + function filterOpts(opt: NonNullable) { + return opt.filter(o => showInternal || !o.hidden) + } + + function emitOptions(opt: NonNullable) { + const trimmedWord = word.startsWith('--') ? word.slice(2) : word.startsWith('-') ? word.slice(1) : word + const names = filterOpts(opt).map(x => x.name).filter(x => x.startsWith(trimmedWord)) + const alreadyAddedOpts = new Set( + words.slice(0, -1) + .filter(x => x.startsWith('--')) + .map(x => x.slice(2)) + .filter(x => !opt.find(o => o.name === x)?.allowMultiple) + ) + + return emitCompletions(names.filter(x => !alreadyAddedOpts.has(x)).map(x => `--${x}`)) + } + + const p = parseArgsForCompletion(args.slice(2), cmd) + if (p === false) { + return + } + + if (p === undefined) { + // we can only complete options now + const opts = cmd.options + if (!opts) { + return + } + + return emitOptions(opts) + } else { + if (p.kind !== 'option' && cmd.options && word.startsWith('-')) { + return emitOptions(cmd.options) + } + + // complete input to arg/opt + if (isFileType(p.type)) { + const cwd = process.cwd() + const files = await glob(getFs(), cwd, [...p.type[fileTypeSym]].map(x => `**/*${x}`), ['node_modules']) + const previous = new Set(words) + + // TODO: these need to be filtered to a single level + const prefixMatched = files.map(x => path.relative(cwd, x)) + .filter(x => !previous.has(x)) + .filter(f => f.startsWith(word)) + + if (prefixMatched.length === 0 && !word && cmd.options && p.kind !== 'option' && (p.allowMultiple || p.optional)) { + return emitOptions(cmd.options) + } + + return emitCompletions(prefixMatched) + } else if (isEnumType(p.type)) { + const values = [...p.type[enumTypeSym]].filter(x => typeof x === 'string') as string[] + return emitCompletions(values.filter(x => x.startsWith(word))) + } + + // TODO: auto-complete symbols + + if (p.kind !== 'option' && (p.allowMultiple || p.optional) && !word && cmd.options) { + return emitOptions(cmd.options) + } + } +} + +function emitCompletions(comps: string[]) { + if (comps.length === 0) { + return + } + + return new Promise(r => process.stdout.write(comps.join('\n'), () => r())) +} + +const unescape = (w: string) => w.charAt(0) === '\'' + ? w.replace(/^'|'$/g, '') + : w.replace(/\\ /g, ' ') \ No newline at end of file diff --git a/src/cli/config.ts b/src/cli/config.ts new file mode 100644 index 0000000..15fea9d --- /dev/null +++ b/src/cli/config.ts @@ -0,0 +1,102 @@ +import { getFs } from '../execution' +import { tryReadJson, tryReadJsonSync } from '../utils' +import { getUserConfigFilePath } from '../workspaces' + +let config: Record +let pendingConfig: Promise> +function _readConfig(): Promise> | Record { + if (config) { + return config + } + + if (pendingConfig) { + return pendingConfig + } + + return pendingConfig = tryReadJson(getFs(), getUserConfigFilePath()).then(val => { + val ??= {} + return config = val as any + }) +} + +let pendingWrite: Promise | undefined +function _writeConfig(conf: Record) { + config = conf + + const write = () => getFs().writeFile(getUserConfigFilePath(), JSON.stringify(conf, undefined, 4)) + + if (pendingWrite) { + const tmp = pendingWrite + .finally(write) + .finally(() => { + if (pendingWrite !== tmp) { + return + } + pendingWrite = undefined + }) + return pendingWrite = tmp + } + + const tmp = write().finally(() => { + if (pendingWrite !== tmp) { + return + } + pendingWrite = undefined + }) + return pendingWrite = tmp +} + +async function readConfig(): Promise> { + return (await tryReadJson(getFs(), getUserConfigFilePath())) ?? {} +} + +async function writeConfig(conf: Record): Promise { + await getFs().writeFile(getUserConfigFilePath(), JSON.stringify(conf, undefined, 4)) +} + +function readConfigSync(): Record { + return tryReadJsonSync(getFs(), getUserConfigFilePath()) ?? {} +} + +function getValue(val: any, key: string) { + const parts = key.split('.') + while (parts.length > 0) { + const k = parts.shift()! + val = val?.[k] + } + + return val +} + +export function readKeySync(key: string): T | undefined { + return getValue(readConfigSync(), key) +} + +export async function readKey(key: string): Promise { + return getValue(await readConfig(), key) +} + +export async function setKey(key: string, value: any) { + const parts = key.split('.') + const config = await readConfig() + let val: any = config + while (parts.length > 1) { + const k = parts.shift()! + if (val[k] === undefined) { + val[k] = {} + } else if (typeof val[k] !== 'object') { + throw new Error(`Found non-object type while setting key "${key}" at access "${k}": ${typeof val[k]}`) + } + + val = val[k] + } + + const oldValue = val[parts[0]] + val[parts[0]] = value + await writeConfig(config) + + return oldValue +} + +// synapse.cli.suggestions -> false +// Need settings to change where test/deploy/synth logs go diff --git a/src/cli/daemon.ts b/src/cli/daemon.ts new file mode 100644 index 0000000..07ef3af --- /dev/null +++ b/src/cli/daemon.ts @@ -0,0 +1,243 @@ +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import * as net from 'node:net' +import * as child_process from 'node:child_process' +import { listenAll, getLogger } from '../logging' +import { executeCommand } from './commands' +import { getSocketsDirectory } from '../workspaces' + +const getSocketPath = () => path.resolve(getSocketsDirectory(), 'daemon.sock') + +interface IpcEventMessage { + readonly type: 'event' + readonly data: any +} + +interface IpcStdoutMessage { + readonly type: 'stdout' + readonly data: string | Buffer +} + +interface IpcErrorMessage { + readonly type: 'error' + readonly data: any +} + +type IpcMessage = IpcEventMessage | IpcStdoutMessage | IpcErrorMessage + +interface IpcCommand { + name: string + args: string[] +} + +export async function startServer() { + const server = net.createServer() + + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(getSocketPath(), () => { + server.removeListener('error', reject) + resolve() + }) + }) + + async function shutdown() { + [...sockets].forEach(s => s.end()) + await new Promise((resolve, reject) => { + server.close(err => err ? reject(err) : resolve()) + }) + process.exit(0) + } + + async function handleRequest(command: IpcCommand) { + try { + if (command.name === 'shutdown') { + return await shutdown() + } + + await executeCommand(command.name, command.args) + + return { + type: 'stdout', + data: '', + } + } catch (e) { + return { + type: 'error', + data: { + ...(e as any), + name: (e as any).name, + message: (e as any).message, + stack: (e as any).stack, + } + } + } + } + + const sockets = new Set() + + function sendMessage(socket: net.Socket, message: string | Buffer) { + return new Promise((resolve, reject) => { + socket.write(message, err => err ? reject(err) : resolve()) + }) + } + + async function broadcast(msg: string | Buffer) { + await Promise.all([...sockets].map(s => sendMessage(s, msg))) + } + + async function broadcastEvent(ev: any) { + await broadcast(JSON.stringify({ type: 'event', data: ev } satisfies IpcEventMessage) + '\n') + } + + process.stdout.on('data', d => getLogger().raw(d)) + + listenAll(getLogger(), broadcastEvent) + + server.on('connection', socket => { + sockets.add(socket) + + socket.on('end', () => { + sockets.delete(socket) + }) + + socket.on('data', async d => { + const cmd = JSON.parse(d.toString('utf-8')) as IpcCommand + const resp = await handleRequest(cmd) + socket.write(JSON.stringify(resp) + '\n') + }) + + // socket.on('error', ...) + }) + + // server.on('close', async () => { + // await shutdown() + // }) + + process.send?.({ status: 'ready' }) +} + +async function startDaemon(daemonModule: string) { + const proc = child_process.fork( + path.resolve(__dirname, '..', 'runtime', 'entrypoint.js'), + [daemonModule], + { + stdio: 'ignore', + detached: true, + } + ) + + await new Promise((resolve, reject) => { + proc.on('error', reject) + proc.on('message', ev => { + if (typeof ev === 'object' && !!ev && 'status' in ev && ev.status === 'ready') { + resolve() + } + }) + }) + + proc.unref() + proc.disconnect() +} + +async function startDaemonIfDown(daemonModule: string, socketPath = getSocketPath()) { + try { + await fs.access(socketPath, fs.constants.F_OK) + + return await openSocket(socketPath) + } catch(e) { + await startDaemon(daemonModule) + + return openSocket(socketPath) + } +} + +async function openSocket(socketPath: string) { + const socket = await new Promise((resolve, reject) => { + function complete(err?: Error) { + s.removeListener('error', complete) + s.removeListener('ready', complete) + + if (err) { + reject(err) + } else { + resolve(s) + } + } + + const s = net.connect(socketPath) + s.once('ready', complete) + s.once('error', complete) + }) + + return socket +} + +export async function connect(daemonModule: string) { + const socket = await startDaemonIfDown(daemonModule) + + async function runCommand(name: string, args: string[]) { + const p = new Promise((resolve, reject) => { + function complete(err?: Error) { + socket.removeListener('exit', exit) + socket.removeListener('data', handleData) + + if (err) { + reject(err) + } else { + resolve() + } + } + + let buffer = '' + async function handleData(d: Buffer) { + buffer += d.toString('utf-8') + + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + + for (const l of lines) { + const msg = JSON.parse(l) as IpcMessage + + if (msg.type === 'error') { + complete(msg.data) + } else if (msg.type === 'stdout') { + complete() + } else { + getLogger().emit(msg.data.type, msg.data) + } + } + } + + function exit() { + complete(new Error(`Socket closed unexpectedly`)) + } + + socket.on('exit', exit) + socket.on('data', handleData) + }) + + await new Promise(async (resolve, reject) => { + socket.write( + JSON.stringify({ name, args } satisfies IpcCommand), + err => err ? reject(err) : resolve() + ) + }) + + return p + } + + async function shutdown() { + return runCommand('shutdown', []) + } + + async function dispose() { + await new Promise((resolve, reject) => socket.end(resolve)) + } + + return { + runCommand, + shutdown, + dispose, + } +} \ No newline at end of file diff --git a/src/cli/index.ts b/src/cli/index.ts new file mode 100644 index 0000000..81871f3 --- /dev/null +++ b/src/cli/index.ts @@ -0,0 +1,322 @@ +#!/usr/bin/env node + +import * as cs from '..' +import * as path from 'node:path' +import { getLogger } from '../logging' +import { LogLevel, logToFile, logToStderr, purgeOldLogs, validateLogLevel } from './logger' +import { CancelError, runWithContext, setContext } from '../execution' +import { RenderableError, colorize, getDisplay, printJson, printLine } from './ui' +import { showUsage, executeCommand, runWithAnalytics, removeInternalCommands } from './commands' +import { getCiType } from '../utils' +import { resolveProgramBuildTarget } from '../workspaces' +import { devLoader } from '../runtime/nodeLoader' +import { readFileSync } from 'node:fs' + +async function _main(argv: string[]) { + if (argv.length === 0) { + return showUsage() + } + + const [cmd, ...params] = argv + + await runWithAnalytics(cmd, async () => { + await executeCommand(cmd, params) + }).finally(() => cs.shutdown()) +} + +function isProbablyRelativePath(arg: string) { + if (arg[0] !== '.') return false + + if (arg[1] === '/' || (arg[1] === '.' && arg[2] === '/')) { + return true + } + + if (process.platform === 'win32') { + return arg[1] === '\\' || (arg[1] === '.' && arg[2] === '\\') + } + + return false +} + +function isMaybeCodeFile(arg: string) { + if (path.isAbsolute(arg)) return true + if (isProbablyRelativePath(arg)) return true + if (arg.length <= 3) return false + + switch (path.extname(arg)) { + case '': + return false + + case '.js': + case '.cjs': + case '.mjs': + return true + + case '.ts': + case '.cts': + case '.mts': + return true + } + + return false +} + +function getLogLevel(): LogLevel | 'off' | undefined { + const envVar = process.env['SYNAPSE_LOG'] + if (!envVar) { + return + } + + if (envVar === 'off') { + return envVar + } + + return validateLogLevel(envVar.toLowerCase()) +} + +const isSea = !!process.env['BUILDING_SEA'] +const revision = process.env['GITHUB_SHA'] +const isProdBuild = process.env.SYNAPSE_ENV === 'production' + +let semver = '0.0.1' +if (isSea) { + const pkgPath = process.env.CURRENT_PACKAGE_DIR + ? path.resolve(process.env.CURRENT_PACKAGE_DIR, 'package.json') + : path.resolve(__dirname, '..', 'package.json') + + const pkgJson = JSON.parse(readFileSync(pkgPath, 'utf-8')) + semver = pkgJson.version +} + +export function main(...args: string[]) { + const arg0 = args[0] + if (arg0 === '--version') { + if (args[1] === '--json') { + const version = { semver, revision } + process.stdout.write(JSON.stringify(version, undefined, 4) + '\n') + } else { + const includeRevision = !isProdBuild || args[1] === '--revision' + const version = `synapse ${semver}${(revision && includeRevision) ? `-${revision}` : ''}` + process.stdout.write(version + '\n') + } + return + } + + if (process.env['SYNAPSE_USE_DEV_LOADER'] && isSea) { + process.argv.splice(1, 1) + delete process.env['SYNAPSE_USE_DEV_LOADER'] + + return devLoader(arg0) + } + + // TODO: if we didn't get a command _and_ `stdin` is piped, parse it as a `.ts` file + + + if (isSea) { + // XXX: this is done for typescript/esbuild + // It seems like `process.argv[0]` isn't resolved? + const execPath = process.platform !== 'darwin' + ? require('node:fs').realpathSync(process.argv[0]) + : process.argv[0] + + globalThis.__filename = path.resolve(path.dirname(path.dirname(execPath)), 'dist', 'cli.js') + globalThis.__dirname = path.dirname(globalThis.__filename) + + // XXX: mostly used as a fallback for "standalone" installs + if (!process.env['SYNAPSE_INSTALL']) { + let currentDir = globalThis.__dirname + const fs = require('node:fs') as typeof import('node:fs') + while (true) { + if (fs.existsSync(path.resolve(currentDir, 'config.json'))) { + process.env['SYNAPSE_INSTALL'] = currentDir + break + } + + const next = path.dirname(currentDir) + if (next === currentDir || next === '.') { + break + } + currentDir = next + } + } + } + + const selfPath = globalThis.__filename ?? __filename + const selfBuildType = isSea ? 'sea' as const : undefined + + if (isSea && arg0 && arg0.startsWith('sea-asset:')) { + process.argv.splice(isSea ? 1 : 2, 1) + + return runWithContext({ selfPath, selfBuildType }, () => cs.runUserScript(arg0)) + } + + // `pointer:` has a length of 72 + // `pointer::` has a length of 137 + if (arg0 && (arg0.length === 72 || arg0.length === 137) && arg0.startsWith('pointer:')) { + process.argv.splice(isSea ? 1 : 2, 1) + + return resolveProgramBuildTarget(process.cwd()).then(buildTarget => { + return runWithContext({ selfPath, selfBuildType, buildTarget }, () => cs.runUserScript(arg0)) + }) + } + + // TODO: -p (print) and -e (eval) + if (arg0 && isMaybeCodeFile(arg0)) { + process.argv.splice(isSea ? 1 : 2, 1) + + const fs = require('node:fs') as typeof import('node:fs') + const resolved = fs.realpathSync(path.resolve(arg0)) + setContext({ selfPath, selfBuildType }) + + return cs.runUserScript(resolved) + } + + Error.stackTraceLimit = 100 + process.on('uncaughtException', e => getLogger().error(e)) + process.on('unhandledRejection', e => getLogger().error(e)) + + // if (args.includes('--inspect')) { + // const inspector = require('node:inspector') as typeof import('node:inspector') + // inspector.open() + // inspector.waitForDebugger() + // } + + function tryGracefulExit(exitCode = process.exitCode) { + let loops = 0 + + function loop() { + const activeResources = (process as any).getActiveResourcesInfo().filter((x: string) => x !== 'TTYWrap') + if (activeResources.length === 0) { + process.exit(exitCode) + } else { + setTimeout(loop, Math.min(loops++, 10)).unref() + } + } + + loop() + } + + async function runWithLogger() { + purgeOldLogs().catch(e => console.error('Failed to purge logs', e)) + + const isCi = !!getCiType() + const logLevel = getLogLevel() + const disposable = logLevel !== 'off' + ? isCi ? logToStderr(getLogger(), logLevel) : logToFile(getLogger(), logLevel) + : undefined + + let didThrow = false + + function handleUnknownError(e: unknown) { + try { + if (process.stdout.isTTY) { + printLine(colorize('red', (e as any).message)) + } else { + process.stderr.write((e as any).message + '\n') + } + + getLogger().error(e) + } catch { + console.error(e) + } + } + + try { + await _main(args) + } catch (e) { + if (e instanceof CancelError) { + didThrow = true + return + } + + if (e instanceof RenderableError) { + try { + await e.render() + } catch (e2) { + getLogger().log('Failed to render', e2) + handleUnknownError(e) + } + } else { + handleUnknownError(e) + } + + didThrow = true + } finally { + await getDisplay().dispose() + await disposable?.dispose() // No more log events will be emitted + + setTimeout(() => { + process.stderr.write(`Forcibly shutting down\n`) + if (process.env['SYNAPSE_DEBUG'] || process.env['CI']) { + process.stderr.write(`Active resources: ${(process as any).getActiveResourcesInfo()}\n`, () => { + process.exit(didThrow ? 1 : undefined) + }) + } + }, 5000).unref() + + if (process.stdout.isTTY) { + tryGracefulExit(didThrow ? 1 : undefined) + } else { + // These unrefs are maybe not needed anymore + process.stdin.unref() + process.stdout.unref?.() + process.stderr.unref() + process.exitCode = process.exitCode || (didThrow ? 1 : 0) + } + } + } + + const ac = new AbortController() + if (process.stdout.isTTY) { + let sigintCount = 0 + process.on('SIGINT', () => { + sigintCount += 1 + if (sigintCount === 1) { + getLogger().debug(`Received SIGINT`) + ac.abort(new CancelError('Received SIGINT')) + } else if (sigintCount === 2) { + getDisplay().dispose().then(() => { + process.exit(1) + }) + } else if (sigintCount === 3) { + process.exit(2) + } + }) + } else { + process.on('SIGINT', () => { + getLogger().debug(`Received SIGINT`) + ac.abort(new CancelError('Received SIGINT')) + }) + } + + return runWithContext({ abortSignal: ac.signal, selfPath, selfBuildType }, runWithLogger).catch(e => { + process.stderr.write((e as any).message + '\n') + process.exit(100) + }) +} + +function seaMain() { + const v8 = require('node:v8') as typeof import('node:v8') + if (!v8.startupSnapshot.isBuildingSnapshot()) { + throw new Error(`BUILDING_SEA was set but we're not building a snapshot`) + } + + v8.startupSnapshot.setDeserializeMainFunction(() => { + const args = process.argv.slice(2) + + return main(...args) + }) +} + +if (isSea) { + if (isProdBuild) { + removeInternalCommands() + } + if (!process.env.SKIP_SEA_MAIN) { + seaMain() + } +} else { + main(...process.argv.slice(2)) +} + +// Note: currently unable to bundle the serialized form of this file due to a cycle somewhere diff --git a/src/cli/install.ps1 b/src/cli/install.ps1 new file mode 100755 index 0000000..6142518 --- /dev/null +++ b/src/cli/install.ps1 @@ -0,0 +1,284 @@ +#!/usr/bin/env pwsh +param( + [String]$Version = "latest", + [String]$InstallLocation, + [Switch]$NoPathUpdate = $false, + [Switch]$NoRegisterInstallation = $false, + [Switch]$Uninstall = $false +); + +if ($env:PROCESSOR_ARCHITECTURE -ne "AMD64") { + Write-Output "Synapse is currently only available for x86 64-bit Windows.`n" + return 1 +} + +$ErrorActionPreference = "Stop" +$RegistryKey = "HKCU:\Software\Microsoft\Windows\CurrentVersion\Uninstall\Synapse" + +# These three environment functions are roughly copied from https://github.com/prefix-dev/pixi/pull/692 +# They are used instead of `SetEnvironmentVariable` because of unwanted variable expansions. +function Publish-Env { + if (-not ("Win32.NativeMethods" -as [Type])) { + Add-Type -Namespace Win32 -Name NativeMethods -MemberDefinition @" +[DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Auto)] +public static extern IntPtr SendMessageTimeout( + IntPtr hWnd, uint Msg, UIntPtr wParam, string lParam, + uint fuFlags, uint uTimeout, out UIntPtr lpdwResult); +"@ + } + $HWND_BROADCAST = [IntPtr] 0xffff + $WM_SETTINGCHANGE = 0x1a + $result = [UIntPtr]::Zero + [Win32.NativeMethods]::SendMessageTimeout($HWND_BROADCAST, + $WM_SETTINGCHANGE, + [UIntPtr]::Zero, + "Environment", + 2, + 5000, + [ref] $result + ) | Out-Null +} + +function Write-Env { + param([String]$Key, [String]$Value) + + $RegisterKey = Get-Item -Path 'HKCU:' + + $EnvRegisterKey = $RegisterKey.OpenSubKey('Environment', $true) + if ($null -eq $Value) { + $EnvRegisterKey.DeleteValue($Key) + } else { + $RegistryValueKind = if ($Value.Contains('%')) { + [Microsoft.Win32.RegistryValueKind]::ExpandString + } elseif ($EnvRegisterKey.GetValue($Key)) { + $EnvRegisterKey.GetValueKind($Key) + } else { + [Microsoft.Win32.RegistryValueKind]::String + } + $EnvRegisterKey.SetValue($Key, $Value, $RegistryValueKind) + } + + Publish-Env +} + +function Get-Env { + param([String] $Key) + + $RegisterKey = Get-Item -Path 'HKCU:' + $EnvRegisterKey = $RegisterKey.OpenSubKey('Environment') + $EnvRegisterKey.GetValue($Key, $null, [Microsoft.Win32.RegistryValueOptions]::DoNotExpandEnvironmentNames) +} + +function Install-Synapse { + param( + [string]$Version + ); + + $IsArchive = $false + $IsUrl = $false + + if ($Version -match "^\d+\.\d+\.\d+$") { + $Version = "synapse-v$Version" + } + elseif ($Version -match "^v\d+\.\d+\.\d+$") { + $Version = "synapse-$Version" + } + elseif ($Version -match "^https:") { + $IsUrl = $true + } + else { + $IsArchive = $Version -ne "latest" + } + + $Arch = "x64" + $DefaultInstallDir = if ($env:SYNAPSE_INSTALL) { $env:SYNAPSE_INSTALL } else { "${Home}\.synapse" } + $InstallDir = if ($InstallLocation) { $InstallLocation } else { $DefaultInstallDir } + mkdir -Force "${InstallDir}" + + try { + Remove-Item "${InstallDir}\app" -Force -Recurse + } catch [System.Management.Automation.ItemNotFoundException] { + # ignore + } catch [System.UnauthorizedAccessException] { + $openProcesses = Get-Process -Name synapse | Where-Object { $_.Path -eq "${InstallDir}\bin\synapse.exe" } + if ($openProcesses.Count -gt 0) { + Write-Output "Failed to remove existing installation because Synapse is running. Stop all processes and try again." + return 1 + } + Write-Output "An unknown error occurred while trying to remove the existing installation" + Write-Output $_ + return 1 + } catch { + Write-Output "An unknown error occurred while trying to remove the existing installation" + Write-Output $_ + return 1 + } + + if (-not $IsArchive) { + $Target = "synapse-windows-$Arch" + $BaseURL = "https://github.com/Cohesible/synapse/releases" + $URL = if ($IsUrl) { $Version } else { + "$BaseURL/$(if ($Version -eq "latest") { "latest/download" } else { "download/$Version" })/$Target.zip" + } + + $ZipPath = "${InstallDir}\$Target.zip" + + Remove-Item -Force $ZipPath -ErrorAction SilentlyContinue + + try { + curl.exe "-#SfLo" "$ZipPath" "$URL" + } catch { + if ($LASTEXITCODE -ne 0) { + Write-Warning "The command 'curl.exe' exited with code ${LASTEXITCODE}`nTrying an alternative download method..." + } + + try { + Invoke-RestMethod -Uri $URL -OutFile $ZipPath + } catch { + Write-Output "Install Failed - could not download $URL" + Write-Output "The command 'Invoke-RestMethod $URL -OutFile $ZipPath' exited with code ${LASTEXITCODE}`n" + return 1 + } + } + } + else { + $ZipPath = $Version + } + + + if (!(Test-Path $ZipPath)) { + Write-Output "Install Failed - could not download $URL" + Write-Output "The file '$ZipPath' does not exist.`n" + return 1 + } + + $Target = "synapse-windows-$Arch" + $Node = "" + + try { + $lastProgressPreference = $global:ProgressPreference + $global:ProgressPreference = 'SilentlyContinue'; + Expand-Archive "$ZipPath" "$InstallDir" -Force + $global:ProgressPreference = $lastProgressPreference + Rename-Item -Path "$InstallDir\$Target" -NewName "app" + if (Test-Path "${InstallDir}\app\bin\synapse.exe") { + $Node = "${InstallDir}\app\bin\synapse.exe" + } + elseif (Test-Path "${InstallDir}\app\bin\node.exe") { # Bootstrap + $Node = "${InstallDir}\app\bin\node.exe" + } + else { + throw "The file '${InstallDir}\app\bin\synapse.exe' does not exist. Download might be corrupt or intercepted by antivirus.`n" + } + } catch { + Write-Output "Install Failed - could not unzip $ZipPath" + Write-Error $_ + return 1 + } + + if (!$IsArchive) { + Remove-Item $ZipPath -Force + } + + $proc = Start-Process -Wait -PassThru -NoNewWindow -FilePath "$Node" -ArgumentList "${InstallDir}\app\dist\install.js","$InstallDir","$InstallDir\app" + if ($proc.ExitCode -ne 0) { + if ($proc.ExitCode -eq 2) { + throw "Failed to finish install due to insufficient permissions. Try running as adminstrator." + } + throw "Failed to finish install" + } + + # if (($LASTEXITCODE -eq 3221225781) -or ($LASTEXITCODE -eq -1073741515)) # STATUS_DLL_NOT_FOUND + # if ($LASTEXITCODE -eq 1073741795) { # STATUS_ILLEGAL_INSTRUCTION + + $installedVersion = "$(& "${InstallDir}\bin\synapse.exe" --version)" + + if (-not $NoRegisterInstallation) { + $rootKey = $null + try { + $rootKey = New-Item -Path $RegistryKey -Force + New-ItemProperty -Path $RegistryKey -Name "Publisher" -Value "Cohesible, Inc." -PropertyType String -Force | Out-Null + # New-ItemProperty -Path $RegistryKey -Name "EstimatedSize" -Value 0 -PropertyType DWord -Force | Out-Null + # $currentDate = Get-Date -Format FileDate + # New-ItemProperty -Path $RegistryKey -Name "InstallDate" -Value $currentDate -PropertyType String -Force | Out-Null + New-ItemProperty -Path $RegistryKey -Name "DisplayName" -Value "Synapse" -PropertyType String -Force | Out-Null + # Set-ItemProperty -Path $RegistryKey -Name "DisplayVersion" -Value "${installedVersion}" -PropertyType String -Force | Out-Null + New-ItemProperty -Path $RegistryKey -Name "InstallLocation" -Value "${InstallDir}" -PropertyType String -Force | Out-Null + # New-ItemProperty -Path $RegistryKey -Name "DisplayIcon" -Value "$InstallDir\bin\synapse.exe" -PropertyType String -Force | Out-Null + + # New-ItemProperty -Path $RegistryKey -Name "NoModify" -Value 1 -PropertyType DWord -Force | Out-Null + # New-ItemProperty -Path $RegistryKey -Name "NoRepair" -Value 1 -PropertyType DWord -Force | Out-Null + try { + curl.exe "-sfLo" "$InstallDir\install.ps1" "https://synap.sh/install.ps1" + } catch { + try { + Invoke-RestMethod -Uri "https://synap.sh/install.ps1" -OutFile "$InstallDir\install.ps1" + } catch {} + } + + New-ItemProperty -Path $RegistryKey -Name "UninstallString" -Value "powershell -c `"& `'$InstallDir\install.ps1`' -Uninstall -PauseOnError`"" -PropertyType String -Force | Out-Null + New-ItemProperty -Path $RegistryKey -Name "QuietUninstallString" -Value "powershell -c `"& `'$InstallDir\install.ps1`' -Uninstall -PauseOnError`"" -PropertyType String -Force | Out-Null + } catch { + Write-Output "Failed to install to registry:" + Write-Output $_ + if ($rootKey -ne $null) { + Remove-Item -Path $RegistryKey -Force + } + } + } else { + Write-Output "Skipped registering installation`n" + } + + try { + $cmd = Get-Command synapse + $previousInstall = Split-Path -Parent $cmd.Source + } catch {} + + $Path = (Get-Env -Key "Path") -split ';' + if ($previousInstall) { + $Path = $Path | where { $_ -ne $previousInstall } + } + if ($Path -notcontains "$InstallDir\bin") { + if (-not $NoPathUpdate) { + $Path += "$InstallDir\bin" + Write-Env -Key 'Path' -Value ($Path -join ';') + # $env:PATH = ($Path -join ';'); + } else { + Write-Output "Skipped adding '${InstallDir}\bin' to %PATH%`n" + } + } + + $C_RESET = [char]27 + "[0m" + $C_GREEN = [char]27 + "[1;32m" + + Write-Output "" + Write-Output "${C_GREEN}Installed ${installedVersion}${C_RESET}" + if (!$previousInstall) { + Write-Output "Restart your editor/terminal and type `"synapse`" to get started`n" + } elseif ($previousInstall -ne "${InstallDir}\bin") { + Write-Output "The install directory has changed. Restart your editor/terminal to apply these changes.`n" + } + + $LASTEXITCODE = 0; +} + +function Uninstall-Synapse { + Write-Output "Uninstalling..." + + $cmd = Get-Command synapse + $previousInstall = Split-Path -Parent $cmd.Source + $Path = (Get-Env -Key "Path") -split ';' + $Path = $Path | where { $_ -ne $previousInstall } + Write-Env -Key 'Path' -Value ($Path -join ';') + + $installDir = Get-ItemPropertyValue -Path $RegistryKey -Name "InstallLocation" + Remove-Item -Force -Recurse $installDir + + Remove-Item -Path $RegistryKey -Force +} + +if ($Uninstall -eq $true) { + Uninstall-Synapse +} else { + Install-Synapse -Version $Version +} diff --git a/src/cli/install.sh b/src/cli/install.sh new file mode 100755 index 0000000..37e43d7 --- /dev/null +++ b/src/cli/install.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${OS:-} = Windows_NT ]]; then + echo 'error: install synapse using Windows Subsystem for Linux or use the Powershell script' + exit 1 +fi + +Color_Off='' +Red='' +Green='' +Dim='' + +if [[ -t 1 ]]; then + Color_Off='\033[0m' + Red='\033[0;31m' + Green='\033[0;32m' + Dim='\033[0;2m' +fi + +error() { + echo -e "${Red}error${Color_Off}:" "$@" >&2 + exit 1 +} + +info() { + echo -e "${Dim}$@ ${Color_Off}" +} + +info_log() { + echo -e "${Dim}$@ ${Color_Off}" >&2 +} + +success() { + echo -e "${Green}$@ ${Color_Off}" +} + +# Used for script generation +DownloadUrl='' +if [ -n "$DownloadUrl" ]; then + set -- "$DownloadUrl" "$@" +fi + +InstallToProfile=${SYNAPSE_INSTALL_TO_PROFILE:-true} + +install_dir=${SYNAPSE_INSTALL:-$HOME/.synapse} +app_dir=$install_dir/app + +if [[ ! -d "$app_dir" ]]; then + mkdir -p "$app_dir" || + error "Failed to create install directory \"$app_dir\"" +fi + +get_target() { + case $(uname -ms) in + 'Darwin x86_64') + # Is this process running in Rosetta? + # redirect stderr to devnull to avoid error message when not running in Rosetta + if [[ $(sysctl -n sysctl.proc_translated 2>/dev/null) = 1 ]]; then + info_log "Your shell is running in Rosetta 2. Downloading synapse for $target instead" + echo darwin-aarch64 + else + echo darwin-x64 + fi + ;; + 'Darwin arm64') + echo darwin-aarch64 + ;; + 'Linux aarch64' | 'Linux arm64') + error "ARM on Linux is not supported yet" # FIXME + echo linux-aarch64 + ;; + 'Linux x86_64' | *) + echo linux-x64 + ;; + esac +} + +install_from_archive() { + strip=0 + case $(basename $1) in + *.xz ) flags=-Jxf;; + *.gz|*.tgz ) flags=-zxf;; + *.zip ) + strip=1 + flags=-zxf + ;; + esac + + tar --strip=$strip $flags "$1" -C "$app_dir" || error 'Failed to extract synapse' + + sea="$app_dir/bin/synapse" + + if [[ -f "$sea" ]]; then + node=$sea + else + node="$app_dir/bin/node" + fi + + chmod +x "$node" + + if [ "$InstallToProfile" = true ]; then + shell=${SHELL:-sh} + "$node" "$app_dir/dist/install.js" "$install_dir" "$app_dir" $(basename "$shell") + else + echo 'Skipping profile install' + "$node" "$app_dir/dist/install.js" "$install_dir" "$app_dir" + fi + + installedVersion=$("$install_dir"/bin/synapse --version) + success "Installed $installedVersion" +} + +if [[ $# -gt 0 ]]; then + if [[ $# -gt 1 ]]; then + install_dir="$2" + app_dir="$install_dir/app" + + if [[ ! -d "$app_dir" ]]; then + mkdir -p "$app_dir" || + error "Failed to create install directory \"$app_dir\"" + fi + + fi + + if [[ "$1" =~ ^https: ]]; then + target=$(get_target) + if [[ "$target" =~ ^darwin ]]; then + tarball_basename=synapse-$target.zip + else + tarball_basename=synapse-$target.tgz + fi + + tarball_path=$app_dir/$tarball_basename + + curl --fail --location --progress-bar --output "$tarball_path" "$1" || error "Failed to download synapse from \"$1\"" + + install_from_archive "$tarball_path" + rm -r "$tarball_path" + else + install_from_archive $1 + fi + + exit 0 +fi + + +target=$(get_target) +if [[ "$target" =~ ^darwin ]]; then + tarball_basename=synapse-$target.zip +else + tarball_basename=synapse-$target.tgz +fi + +GITHUB=${GITHUB-"https://github.com"} + +github_repo="$GITHUB/Cohesible/synapse" + +if [[ $# = 0 ]]; then + synapse_uri=$github_repo/releases/latest/download/$tarball_basename +else + synapse_uri=$github_repo/releases/download/$1/$tarball_basename +fi + + +tarball_path=$app_dir/$tarball_basename + +curl --fail --location --progress-bar --output "$tarball_path" "$synapse_uri" || + error "Failed to download synapse from \"$synapse_uri\"" + +install_from_archive "$tarball_path" + +rm -r "$tarball_path" + +if [[ ! -z "$GITHUB_ENV" ]]; then + info "Probably in a GitHub runner, installing to workflow paths" + echo "SYNAPSE_INSTALL=$install_dir" >> "$GITHUB_ENV" + echo "$install_dir/bin" >> "$GITHUB_PATH" +fi diff --git a/src/cli/install.ts b/src/cli/install.ts new file mode 100644 index 0000000..bbcd9bf --- /dev/null +++ b/src/cli/install.ts @@ -0,0 +1,200 @@ +// This file will called by `install.sh` to finish setting things up + +import * as os from 'node:os' +import * as path from 'node:path' +import * as child_process from 'node:child_process' +import { createLocalFs, ensureDir } from '../system' +import { readKey, setKey } from './config' +import { createInstallCommands, installToUserPath, isSupportedShell } from '../pm/publish' +import { Export } from 'synapse:lib' +import { makeExecutable } from '../utils' + +new Export({}) // XXX: adding this so the bundle gets optimized + +export async function main(installDir: string, pkgDir: string, shell?: string) { + const resolvedInstallDir = path.resolve(installDir) + const resolvedPkgDir = path.resolve(pkgDir) + process.env['SYNAPSE_INSTALL'] ||= resolvedInstallDir + + if (resolvedPkgDir.startsWith(resolvedInstallDir)) { + console.log(`Installing to "${resolvedInstallDir}"`) + } else { + console.log(`Installing "${resolvedPkgDir}" to "${resolvedInstallDir}"`) + } + + try { + await finishInstall(resolvedInstallDir, resolvedPkgDir, shell) + } catch (e) { + console.log('Install failed!', e) + process.exitCode = 1 + + if ((e as any).code === 'EPERM' && process.platform === 'win32') { + process.exitCode = 2 + } + } +} + +function getExec(name: string) { + return process.platform === 'win32' ? `${name}.exe` : name +} + +function getExecutablePath(pkgDir: string, name: string) { + return path.resolve(pkgDir, 'bin', getExec(name)) +} + +function getToolPath(pkgDir: string, name: string) { + return path.resolve(pkgDir, 'tools', getExec(name)) +} + +function getCwdVar() { + return process.platform === 'win32' ? '%~dp0' : '$SCRIPT_DIR' +} + +function getVarArgs() { + return process.platform === 'win32' ? '%*' : '"$@"' +} + +async function createWindowsExecutableScript(cliRelPath: string, nodeRelPath: string, binDir: string) { + const nodeArgs = ['"%CLI_PATH%"', '%*'] + + const cwd = '%~dp0' + const contents = ` +@ECHO OFF + +SET "SYNAPSE_PATH=%0" +SET "NODE_PATH=${cwd}/${nodeRelPath}" +SET "CLI_PATH=${cwd}/${cliRelPath}" + +"%NODE_PATH%" ${nodeArgs.join(' ')} +`.trim() + + const dest = path.resolve(binDir, 'synapse.cmd') + const fs = createLocalFs() + await fs.writeFile(dest, contents, { mode: 0o755 }) + + return dest +} + +type PackageType = 'sea' | 'script' + +async function createExecutableScript(pkgDir: string, installDir: string, pkgType: PackageType = 'script') { + const fs = createLocalFs() + const binDir = path.resolve(installDir, 'bin') + const seaPath = getExecutablePath(pkgDir, 'synapse') + const nodeRelPath = path.relative(binDir, getExecutablePath(pkgDir, 'node')) + const cliRelPath = path.relative(binDir, path.resolve(pkgDir, 'dist', 'cli.js')) + if (await fs.fileExists(seaPath)) { + pkgType = 'sea' + } + + const nodeArgs: string[] = [] + + if (process.platform === 'win32') { + if (pkgType !== 'sea') { + return createWindowsExecutableScript(cliRelPath, nodeRelPath, binDir) + } + + const dest = path.resolve(binDir, 'synapse.exe') + + try { + await fs.link(seaPath, dest, { symbolic: true, typeHint: 'file' }) + } catch (e) { + if ((e as any).code !== 'EPERM') { + throw e + } + + // Windows "junctions" don't require admin + // XXX: need to rename `terraform.exe` so we don't shadow existing installs + await fs.link(path.dirname(seaPath), path.dirname(dest), { symbolic: true, typeHint: 'junction' }) + } + + return dest + } + + const dest = path.resolve(binDir, 'synapse') + + if (pkgType !== 'sea') { + nodeArgs.push(`"$SCRIPT_DIR/${cliRelPath}"`, '"$@"') + + const executable = ` +#!/bin/sh + +export SYNAPSE_PATH="$0" +SCRIPT_DIR=$(dirname -- "$( readlink -f -- "$0"; )"); + +exec "$SCRIPT_DIR/${nodeRelPath}" ${nodeArgs.join(' ')} + `.trim() + + await fs.writeFile(dest, executable, { mode: 0o755 }) + } else { + await makeExecutable(seaPath) + await fs.link(seaPath, dest, { symbolic: true }) + } + + return dest +} + +async function finishInstall(installDir: string, pkgDir: string, shell?: string) { + const fs = createLocalFs() + const newTfPath = getToolPath(pkgDir, 'terraform') + const newEsbuildPath = getToolPath(pkgDir, 'esbuild') + + const synapse = await createExecutableScript(pkgDir, installDir) + + + // XXX: needed until the packages are hosted somewhere + // Ideally this should be a fallback for hosted versions rather than an override + const overrides: Record = { synapse: pkgDir } + const builtinPackagesDir = path.resolve(pkgDir, 'packages') + for (const f of await fs.readDirectory(builtinPackagesDir)) { + overrides[f.name] = path.resolve(builtinPackagesDir, f.name) + } + + const oldOverrides = await readKey('projectOverrides') + await setKey('projectOverrides', { ...overrides, ...oldOverrides }) + + await Promise.all([ + setKey('terraform.path', newTfPath), + makeExecutable(newTfPath), + ]) + + await Promise.all([ + setKey('esbuild.path', newEsbuildPath), + makeExecutable(newEsbuildPath), + ]) + + await setKey('typescript.libDir', path.resolve(pkgDir, 'dist')) + + // Make sure things work + // `shell: true` is required to spawn `.cmd` files + const out = child_process.spawnSync(synapse, ['--version'], { encoding: 'utf-8', shell: synapse.endsWith('.cmd') }) + if (out.status !== 0) { + if (out.status === null && out.error) { + throw new Error(`Failed to run Synapse`, { cause: out.error }) + } + throw new Error(`Failed to run Synapse: ${out.stdout + '\n' + out.stderr}`) + } + + if (process.platform === 'win32') { + return + } + + const completionsScriptPath = path.resolve(installDir, 'completions', 'synapse.sh') + + await fs.writeFile( + completionsScriptPath, + await fs.readFile(path.resolve(pkgDir, 'dist', 'completions.sh')) + ) + + if (!shell || !isSupportedShell(shell)) { + console.log('Synapse was not added to your PATH.') + console.log('You will have to manually update relevant config files with the following:') + const commands = createInstallCommands(installDir, false, completionsScriptPath) + for (const c of commands) { + console.log(` ${c}`) + } + return + } + + await installToUserPath(shell, installDir) +} diff --git a/src/cli/logger.ts b/src/cli/logger.ts new file mode 100644 index 0000000..bd7b41c --- /dev/null +++ b/src/cli/logger.ts @@ -0,0 +1,255 @@ +import * as path from 'node:path' +import * as perf from 'node:perf_hooks' +import { formatWithOptions } from 'node:util' +import { LogEvent, Logger, PerfDetail } from '../logging' +import { FsEntityStats, openHandle } from '../system' +import { memoize } from '../utils' +import { getExecutionId, getFs } from '../execution' +import { getLogsDirectory } from '../workspaces' +import { isDataPointer } from '../build-fs/pointers' + +const fmtDuration = (val: number, digits = 3) => `${Math.floor(val * Math.pow(10, digits)) / Math.pow(10, digits)} ms` + +// Could also add the symbol `nodejs.util.inspect.custom` +function format(a: any) { + if (isDataPointer(a)) { + if (!a.isResolved()) { + return `DataPointer ` + } + + const { hash, storeHash } = a.resolve() + if (!storeHash) { + return `DataPointer ` + } + + return `DataPointer <${storeHash.slice(0, 12)} ${hash.slice(0, 12)}>` + } + + return a +} + +const print = (...args: any[]) => formatWithOptions({ colors: false, depth: 4 }, ...args.map(format)) + +export function logToFile( + logger: Logger, + logLevel: Exclude = 'debug', + fileName = path.resolve(getLogsDirectory(), `${getExecutionId()}.log`), +) { + // Log entries are buffered in-mem until the file handle is ready + const buffer: string[] = [] + const getHandle = memoize(async () => { + const h = await openHandle(fileName).catch(e => { + console.error(e) + throw e + }) + + while (buffer.length > 0) { + h.write(buffer.shift()!) + } + + return h + }) + + + let handle: Awaited> + function enqueue(entry: string) { + if (!handle) { + buffer.push(entry) + getHandle().then(h => handle = h) + } else { + handle.write(entry) + } + } + + const listeners: { dispose: () => void }[] = [] + + listeners.push(logger.onPerf(ev => { + const displayName = ev.taskType ? `${ev.taskType} (${ev.taskName})` : `${ev.taskName}` + + if (ev.duration > (ev.slowThreshold ?? 1000)) { + const timestamp = ev.timestamp.toISOString() + enqueue(`${timestamp} [PERF] ${print(displayName, fmtDuration(ev.duration))}\n`) + } + })) + + listeners.push(logger.onLog(ev => { + if (ev.level === 'raw') { // This is for terraform logs + enqueue(ev.args[0]) + } else { + if (compareLogLevel(ev.level, logLevel) > 0) { + return + } + + // Timestamps are always printed using UTC + const timestamp = ev.timestamp.toISOString() + const entry = `${timestamp} [${ev.level.toUpperCase()}] ${print(...ev.args)}\n` + enqueue(entry) + } + })) + + async function dispose() { + listeners.forEach(l => l.dispose()) + if (!getHandle.cached && buffer.length === 0) { + return + } + + return getHandle().then(h => h.dispose()) + } + + return { dispose } +} + +export async function listLogFiles() { + const fs = getFs() + const logsDir = getLogsDirectory() + const files = await fs.readDirectory(logsDir).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + return [] + }) + + return files.filter(f => f.type === 'file').map(f => path.resolve(logsDir, f.name)) +} + +export async function getMostRecentLogFile() { + const sorted = await getSortedLogs() + + return sorted[0]?.filePath as string | undefined +} + +export async function getSortedLogs() { + const files = await listLogFiles() + const stats: (FsEntityStats & { filePath: string })[] = [] + for (const f of files) { + try { + stats.push({ + filePath: f, + ...(await getFs().stat(f)), + }) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } + + } + + return stats.sort((a, b) => b.ctimeMs - a.ctimeMs) +} + +export async function purgeOldLogs() { + const sorted = await getSortedLogs() + for (const f of sorted.slice(25)) { + if (f.filePath.endsWith('analytics.log')) continue // XXX + await getFs().deleteFile(f.filePath).catch(e => { + if ((e as any).code !== 'ENOENT') { + // Failed to delete log file for some reason (possibly opened?) + } + + // Log file was deleted by something else + }) + } +} + +const levels = new Set(['error', 'warn', 'info', 'debug', 'trace']) + +function logLevelOrder(logLevel: Exclude): number { + switch (logLevel) { + case 'error': + return 0 + case 'warn': + return 1 + case 'info': + return 2 + case 'debug': + return 3 + case 'trace': + return 4 + } +} + +export type LogLevel = Exclude +export function validateLogLevel(level: string): LogLevel | undefined { + if (levels.has(level)) { + return level as LogLevel + } +} + +function compareLogLevel(a: LogLevel, b: typeof a) { + return logLevelOrder(a) - logLevelOrder(b) +} + +export function logToStderr(logger: Logger, logLevel: LogLevel = 'debug') { + const stream = process.stderr + const print = (...args: any[]) => formatWithOptions({ colors: stream.isTTY, depth: 4 }, ...args) + const level = logLevelOrder(logLevel) + + logger.onLog(ev => { + if (ev.level === 'raw') { // This is for terraform logs + stream.write(ev.args[0]) + } else if (logLevelOrder(ev.level) <= level) { + stream.write(`[${ev.level.toUpperCase()}] ${print(...ev.args)}\n`) + } + }) + + logger.onPerf(ev => { + const displayName = ev.taskType ? `${ev.taskType} (${ev.taskName})` : `${ev.taskName}` + + if (ev.duration > (ev.slowThreshold ?? 100)) { + stream.write(`[PERF] ${print(displayName, fmtDuration(ev.duration))}\n`) + } + }) + + logger.onDeploy(ev => { + if (ev.status === 'failed') { + logger.error(`Failed to deploy ${ev.resource}:`, (ev as any).reason) + } + }) + + logger.onTest(ev => { + // XXX: don't write out suite names + if (ev.parentId === undefined) { + return + } + + if (ev.status === 'passed') { + stream.write(`[TEST]: ${ev.name} [passed]\n`) + } else if (ev.status === 'failed') { + stream.write(`[TEST]: ${ev.name} [failed]\n`) + stream.write(print(ev.reason).split('\n').map(x => ` ${x}`).join('\n') + '\n') + } + }) + + logger.onTestLog(ev => { + if (logLevelOrder(ev.level) <= level) { + stream.write(`> (${ev.name}) [${ev.level.toUpperCase()}] ${print(...ev.args)}\n`) + } + }) + + registerDeployPerfHook(logger) + + return { dispose: () => logger.dipose() } +} + +// Translates `deploy` events into timing events +function registerDeployPerfHook(logger: Logger, slowThreshold = 100) { + const marked = new Set() + + return logger.onDeploy(ev => { + const detail: PerfDetail = { taskType: 'deploy', taskName: ev.resource, slowThreshold } + + // FIXME: this can collide if deploying multiple programs as the same time + if (ev.status === 'applying' && !marked.has(ev.resource)) { + perf.performance.mark(`deploy-${ev.resource}`, { detail }) + marked.add(ev.resource) + } + + if (ev.status === 'complete' && marked.has(ev.resource)) { + const markName = `deploy-${ev.resource}` + perf.performance.measure(`deploy-${ev.resource}-complete`, { start: markName, detail }) + perf.performance.clearMarks(markName) + marked.delete(ev.resource) + } + }) +} diff --git a/src/cli/ui.ts b/src/cli/ui.ts new file mode 100644 index 0000000..a22069e --- /dev/null +++ b/src/cli/ui.ts @@ -0,0 +1,1427 @@ +import { getLogger } from '..' +import { createEventEmitter } from '../events' +import { memoize } from '../utils' +import * as nodeUtil from 'node:util' + +const esc = '\x1b' + +const foregroundColors = { + black: 30, + red: 31, + green: 32, + yellow: 33, + blue: 34, + purple: 35, + cyan: 36, + white: 37, + gray: 90, + brightRed: 91, + brightGreen: 92, + brightYellow: 93, + brightBlue: 94, + brightPurple: 95, + brightCyan: 96, + brightWhite: 97, +} + +const backgroundColors = { + black: 40, + red: 41, + green: 42, + yellow: 43, + blue: 44, + purple: 45, + cyan: 46, + white: 47, +} + +const modifiers = { + none: 0, + bold: 1, + underline: 4, +} + +const eightBitColors = { + commentGreen: 65, + orange: 214, + orange2: 215, + paleYellow: 229, + paleCyan: 159, + paleGreen: 194, +} + +export type EightBitColor = keyof typeof eightBitColors +export type AnsiColor = keyof typeof foregroundColors +export type Color = AnsiColor | EightBitColor | 'none' + +export function dim(text: string) { + return `${esc}[2m${text}${esc}[0m` +} + +export function bold(text: string) { + return `${esc}[1m${text}${esc}[0m` +} + +// 8-bit +// ESC[38;5;⟨n⟩m Select foreground color where n is a number from the table below +// ESC[48;5;⟨n⟩m Select background color +// 0- 7: standard colors (as in ESC [ 30–37 m) +// 8- 15: high intensity colors (as in ESC [ 90–97 m) +// 16-231: 6 × 6 × 6 cube (216 colors): 16 + 36 × r + 6 × g + b (0 ≤ r, g, b ≤ 5) +// 232-255: grayscale from dark to light in 24 steps + +// 24-bit +// ESC[38;2;⟨r⟩;⟨g⟩;⟨b⟩ m Select RGB foreground color +// ESC[48;2;⟨r⟩;⟨g⟩;⟨b⟩ m Select RGB background color + +// COLORTERM=truecolor + +export function colorize(color: Color, text: string) { + if (color === 'none') { + return text + } + + if (color in eightBitColors) { + const code = eightBitColors[color as EightBitColor] + + return `${esc}[38;5;${code}m${text}${esc}[0m` + } + + const code = foregroundColors[color as AnsiColor] + + return `${esc}[${code}m${text}${esc}[0m` +} + +function colorizeTemplate( + color: Exclude, + strings: TemplateStringsArray, + values: string[] +) { + const result = [strings[0]] + values.forEach((value, i) => { + result.push(value, strings[i + 1]) + }) + + return colorize(color, result.join('')) +} + +// FIXME: doesn't handle colon-separated commands +const ansiPattern = /\u001b\[[0-9]+m/g + +export function stripAnsi(s: string) { + return s.replace(ansiPattern, '') +} + + +let display: ReturnType +export function getDisplay() { + return display ??= createDisplay() +} + +function swap(arr: any[], i: number, j: number) { + const tmp = arr[i] + arr[i] = arr[j] + arr[j] = tmp +} + +function createSyncWrapper any>>(obj: T) { + interface Operation { + readonly id: number + readonly type: K + readonly args: Parameters + } + + let idCounter = 0 + const queue: Operation[] = [] + const pending = new Set() + + async function processOps() { + while (queue.length > 0) { + const op = queue.shift()! + if (pending.has(op.id)) { + pending.delete(op.id) + await (obj[op.type] as any)(...op.args) + } + } + } + + let processing: Promise | undefined + function startProcessing() { + if (processing) { + return processing + } + + processing = processOps().finally(() => (processing = undefined)) + } + + function cancel(id: number) { + pending.delete(id) + } + + function cancelAll() { + pending.clear() + } + + async function flush() { + await startProcessing() + } + + const wrapped: { [P in keyof T]: (...args: Parameters) => number } = {} as any + for (const [k, v] of Object.entries(obj)) { + const type = k as keyof T + wrapped[type] = (...args: any) => { + const id = idCounter++ + pending.add(id) + queue.push({ id, type, args }) + if (queue.length === 1) { + startProcessing() + } + + return id + } + } + + function createCompositeOp(fn: (writer: typeof wrapped) => void): { cancel: () => void } { + const start = idCounter + fn(wrapped) + const end = idCounter + + function cancel() { + for (let i = start; i < end; i++) { + pending.delete(i) + } + } + + return { cancel } + } + + return { + wrapped, + flush, + cancel, + cancelAll, + createCompositeOp, + } +} + +function createScreenWriter(stream = process.stdout) { + async function waitForDrain() { + await new Promise((r) => stream.once('drain', r)) + } + + async function runAndDrain(fn: (cb: (val: T) => void) => boolean) { + let didDrain = false + const val = await new Promise(resolve => { + didDrain = fn(resolve) + }) + + if (!didDrain) { + await waitForDrain() + } + + return val + } + + async function write(data: string | Buffer) { + const err = await runAndDrain(resolve => stream.write(data, resolve)) + if (err) { + throw err + } + } + + async function cursorTo(x: number, y?: number) { + await runAndDrain((resolve) => stream.cursorTo(x, y, resolve)) + } + + async function moveCursor(dx: number, dy: number) { + await runAndDrain((resolve) => stream.moveCursor(dx, dy, resolve)) + } + + async function clearScreenDown() { + await runAndDrain((resolve) => stream.clearScreenDown(resolve)) + } + + /** + * -1 - to the left from cursor + * 0 - the entire line + * 1 - to the right from cursor + */ + async function clearLine(dir: -1 | 0 | 1 = 0) { + await runAndDrain((resolve) => stream.clearLine(dir, resolve)) + } + + async function clearScreen(fullScreen = false) { + await cursorTo(0, fullScreen ? 0 : 1) + await clearScreenDown() + } + + async function scrollUp(lines = 1) { + await write(`${esc}[${lines}S`) + } + + async function scrollDown(lines = 1) { + await write(`${esc}[${lines}T`) + } + + async function showCursor() { + await write(`${esc}[?25h`) + } + + async function hideCursor() { + await write(`${esc}[?25l`) + } + + async function writeRows(rows: string[], fullScreen = false) { + await clearScreen(fullScreen) + await write(rows.join('\n')) + } + + async function setupScreen(fullScreen = false) { + const pos = await getCursorPosition() + if (pos && pos.row > 1) { + const offset = stream.rows - 1 + await write('\n'.repeat(offset - (fullScreen ? 0 : 1))) + } + } + + const operations = { write, scrollUp, cursorTo, clearScreenDown, clearLine, clearScreen, writeRows, moveCursor, setupScreen, showCursor, hideCursor } + + const { wrapped, flush, cancel, cancelAll, createCompositeOp } = createSyncWrapper(operations) + + const tty = registerTty() + + async function getCursorPosition() { + if (!tty) { + return + } + + const p = new Promise(r => { + const l = tty!.onDeviceStatus(ev => { + l.dispose() + r(ev) + }) + }) + + await write(`${esc}[6n`) + + return p + } + + async function end(data?: string | Uint8Array) { + await flush() + + return new Promise((resolve, reject) => { + data !== undefined ? stream.end(data, resolve) : stream.end(resolve) + }) + } + + return { + ...wrapped, + cancel, + cancelAll, + flush, + end, + createCompositeOp, + getCursorPosition, + tty, + } +} + +interface ViewData { + readonly fullScreen?: boolean + readonly rows: string[] + readonly currentRow: number + readonly pendingWrites: number[] + readonly cursorPosition: { + readonly row: number + readonly column: number + } +} + +enum ControlKey { + Enter = 13, + UpArrow, + DownArrow, + RightArrow, + LeftArrow, +} + +interface SignalEvent { + readonly signal: NodeJS.Signals +} + +// Emits ASCII characters only +interface KeyPressEvent { + readonly key: string | ControlKey +} + +interface DeviceStatusEvent { + readonly row: number + readonly column: number +} + +function registerTty() { + if (!process.stdout.isTTY) { + return + } + + const emitter = createEventEmitter() + + const controlKeys = { + ['A'.charCodeAt(0)]: ControlKey.UpArrow, + ['B'.charCodeAt(0)]: ControlKey.DownArrow, + ['C'.charCodeAt(0)]: ControlKey.RightArrow, + ['D'.charCodeAt(0)]: ControlKey.LeftArrow, + } + + function handleInput(d: Buffer) { + let isEscapeCode = false + + for (let i = 0; i < d.length; i++) { + if (d[i] === 0x03) { + emitter.emit('signal', { signal: 'SIGINT' } satisfies SignalEvent) + } else if (d[i] === 0x1b && d[i+1] === 0x5b) { + i += 1 + isEscapeCode = true + } else if (isEscapeCode) { + isEscapeCode = false + + // Try to parse out the cursor position + if (d[d.length - 1] === 0x52) { // 'R' + const sub = d.subarray(i, d.length - 1) + const sepIndex = sub.indexOf(0x3b) + if (sepIndex === -1) { + break + } + + const row = parseInt(String.fromCharCode(...sub.subarray(0, sepIndex))) - 1 + const column = parseInt(String.fromCharCode(...sub.subarray(sepIndex + 1))) - 1 + if (!isNaN(row) && !isNaN(column)) { + emitter.emit('device-status', { row, column } satisfies DeviceStatusEvent) + } + + break + } + + const key = controlKeys[d[i]] + if (key !== undefined) { + emitter.emit('keypress', { key } satisfies KeyPressEvent) + } + } else if (d[i] > 0x1f && d[i] < 0x7f) { + emitter.emit('keypress', { key: String.fromCharCode(d[i]) } satisfies KeyPressEvent) + } + } + } + + function onDeviceStatus(listener: (ev: DeviceStatusEvent) => void) { + emitter.addListener('device-status', listener) + + return { dispose: () => emitter.removeListener('device-status', listener) } + } + + function onKeyPress(listener: (ev: KeyPressEvent) => void) { + emitter.addListener('keypress', listener) + + return { dispose: () => emitter.removeListener('keypress', listener) } + } + + function onSignal(listener: (ev: SignalEvent) => void) { + emitter.addListener('signal', listener) + + return { dispose: () => emitter.removeListener('signal', listener) } + } + + function dispose() { + emitter.removeAllListeners() + if (process.stdin.isTTY) { + process.stdin.setRawMode(false) + process.stdin.removeListener('data', handleInput) + } + } + + if (process.stdin.isTTY) { + process.stdin.setRawMode(true) + process.stdin.on('data', handleInput) + } + + return { onSignal, onKeyPress, onDeviceStatus, dispose } +} + +interface DisplayRow { + update: (text: string) => void + // "Releases" the row, allowing it to be pushed off the screen + release: (text?: string, emptyDelay?: number) => void + // Removes the row entirely and leaves an empty space + destroy: () => void +} + +export function createDisplay() { + const viewStack: string[] = [] + const views = new Map() + const writer = createScreenWriter() + + function renderView(data: ViewData) { + const start = data.currentRow + const end = start + Math.min(process.stdout.rows - (data.fullScreen ? 0 : 1), data.rows.length) + const rows = data.rows.slice(start, end).map(r => r.slice(0, process.stdout.columns)) + data.pendingWrites.push(writer.writeRows(rows)) + data.pendingWrites.push(writer.cursorTo(data.cursorPosition.column, data.cursorPosition.row)) + } + + // For non-tty displays we only show text from `writeLine` or the final text of a row after `release` + function createOverlayableView() { + let idCounter = 0 + let screenTop: number + let height = process.stdout.rows + let needsSort = false + let perfTime = 0 + let frames = 0 + let drawStart: number + + const getCursorPosition = memoize(() => writer.getCursorPosition()) + + async function _getScreenTop() { + if (screenTop !== undefined) { + return screenTop + } + + if (!process.stdout.isTTY) { + return screenTop = 0 + } + + // XXX: hardcoded delay to make shutdowns faster + // We only need to know the cursor position when we want + // to keep track of the text we've drawn. But that doesn't + // matter if we've finished execution. + await new Promise(r => setTimeout(r, 25).unref()) + + if (disposed) { + return 0 + } + + const cursor = await getCursorPosition() + if (!cursor) { + throw new Error(`No cursor position found`) + } + + if (disposed) { + return 0 + } + + writer.hideCursor() + + return screenTop = cursor.row + } + + const getScreenTop = memoize(_getScreenTop) + + interface Span { + id: number + row?: number + empty?: boolean + open?: boolean + released?: boolean + // drawnPos?: number + text: string + spinner?: Spinner + startTime?: number + forceRemove?: boolean + } + + function getSpanText(span: Span) { + if (!span.spinner) { + return span.text + } + + const delta = span.startTime !== undefined + ? drawStart - span.startTime + : 0 + + const spinnerFrame = getSpinnerFrame(span.spinner, delta) + span.startTime ??= drawStart + + return `${spinnerFrame} ${span.text}` + } + + const spans: Span[] = [] + + process.stdout.on('resize', async () => { + const delta = process.stdout.rows - height + height = process.stdout.rows + if (delta < 0) { + screenTop = Math.max(await getScreenTop() + delta, 0) + redraw() + } + }) + + function shiftUp() { + const idx = spans.findIndex(s => s.row === undefined || s.released) + if (idx === -1) { + return false + } + + // TODO: we can skip writing and sorting when `idx` is 0 and the span + // was in the same row + text on the last draw + + writer.cursorTo(0, 0) + const width = Math.max(spans[0].text.length + 1, process.stdout.columns) + writer.write(getSpanText(spans[idx]).padEnd(width, ' ')) + writer.cursorTo(process.stdout.columns - 1, process.stdout.rows - 1) + + swap(spans, idx, spans.length - 1) + spans.pop() + + return true + } + + function index(row: number) { + return row < 0 ? spans.length + row : row + } + + function sortSpans() { + // const indices = new Map(spans.map((s, i) => [s, i])) + spans.sort((a, b) => { + if (a.row !== undefined && b.row !== undefined) { + return (index(a.row) - index(b.row)) || (a.id - b.id) + } + + if (a.row !== undefined && b.row === undefined) { + return index(a.row) - spans.indexOf(a) + } + + if (a.row === undefined && b.row !== undefined) { + return spans.indexOf(b) - index(b.row) + } + + return a.id - b.id + }) + } + + function addSpan(s: Span) { + if (disposed) { + return + } + + const p = performance.now() + spans.push(s) + needsSort = true + //sortSpans() + redraw() + perfTime += performance.now() - p + } + + // TODO: account for spans taking up multiple lines + + // TODO: track the last 20 or so rows that were pushed off the screen + // This would allows us to redraw the screen correctly on resize + function _redraw() { + if (disposed) { + return + } + + const p = performance.now() + drawStart = p + t = undefined + frames += 1 + + if (needsSort) { + sortSpans() + needsSort = false + } + + let forceRemoveIndex = -1 + while ((forceRemoveIndex = spans.findIndex(x => x.forceRemove)) !== -1) { + spans.splice(forceRemoveIndex, 1) + } + + let isCursorAtBottom = false + function scroll() { + if (!isCursorAtBottom) { + writer.cursorTo(process.stdout.columns - 1, process.stdout.rows - 1) + isCursorAtBottom = true + } + + writer.write('\n') + } + + while (spans.length > (process.stdout.rows - screenTop)) { + if (screenTop === 0) { + if (!shiftUp()) { + break + } else { + sortSpans() + } + } else { + screenTop -= 1 + } + + scroll() + } + + perfTime += performance.now() - p + + writer.cursorTo(0, screenTop) + writer.clearScreenDown() + + const rows = spans.slice(0, process.stdout.rows - screenTop) + writer.write(rows.map(getSpanText).join('\n')) + + if (rows.some(s => s.spinner)) { + t = +setTimeout(_redraw, 25).unref() + } + } + + let t: number | undefined + function redraw() { + if (t !== undefined) { + return + } + + if (screenTop !== undefined) { + t = +setTimeout(_redraw).unref() + } else { + t = +setTimeout(() => getScreenTop().then(_redraw)).unref() + } + } + + function rowFromSpan(s: Span): DisplayRow { + const existing = rows.get(s) + if (existing) { + return existing + } + + function update(text: string) { + if (s.released) { + return + } + + if (s.text !== text) { + s.text = text + redraw() + } + } + + function release(text?: string, emptyDelay?: number) { + if (s.released) { + return + } + + if (text !== undefined) { + s.text = text + } + + s.released = true + redraw() + + if (emptyDelay !== undefined) { + setTimeout(() => { + s.forceRemove = true + redraw() + }, emptyDelay).unref() + } + } + + function destroy() { + if (s.released) { + return + } + + s.released = true + const idx = spans.indexOf(s) + if (idx === -1) { + return + } + + const needsSwap = idx !== spans.length - 1 + if (needsSwap) { + swap(spans, idx, spans.length - 1) + } + + spans[spans.length - 1] = { + id: idCounter++, + empty: true, + text: '', + } + + if (needsSwap) { + needsSort = true + } + + redraw() + } + + const row: DisplayRow = { update, release, destroy } + rows.set(s, row) + + return row + } + + function createSpan(attr: Omit): Span { + return { + id: idCounter++, + ...attr, + } + } + + function findAvailableIndex(allowOpen = false) { + for (let i = spans.length - 1; i >= 0; i--) { + const s = spans[i] + if (s.row !== undefined) { + continue + } else if (s.empty || (allowOpen && s.open)) { + return i + } else { + break + } + } + + return -1 + } + + function findOrCreateSpan(attr: Omit) { + const p = performance.now() + const idx = findAvailableIndex() + if (idx === -1) { + const s: Span = createSpan(attr) + perfTime += performance.now() - p + addSpan(s) + + return s + } + + const s = spans[idx] = createSpan(attr) + + redraw() + perfTime += performance.now() - p + + return s + } + + const rows = new WeakMap() + function createRow(text = '', row = 0, spinner?: Spinner): DisplayRow { + const s = findOrCreateSpan({ text, released: false, row, spinner }) + + return rowFromSpan(s) + } + + function createFooter(text = '') { + return createRow(text, -1) + } + + function write(text: string = '', leaveOpen = true) { + const p = performance.now() + const idx = findAvailableIndex(true) + if (idx === -1) { + perfTime += performance.now() - p + + return addSpan(createSpan({ text, open: leaveOpen })) + } + + if (spans[idx].empty) { + spans[idx] = createSpan({ text, open: leaveOpen }) + } else { + spans[idx].text += text + spans[idx].open = leaveOpen + } + + redraw() + perfTime += performance.now() - p + } + + // Writes text in the non-overlayed screen, potentially scrolling the screen down + function writeLine(text: string = '') { + write(text, false) + } + + let disposed = false + async function dispose() { + if (disposed) { + return + } + + clearTimeout(t) + disposed = true + + if (getCursorPosition.cached) { + await getCursorPosition() + } + + const p = performance.now() + + // Erase anything we've already drawn + if (screenTop !== undefined) { + writer.cursorTo(0, screenTop) + writer.clearScreenDown() + } + + if (needsSort) { + sortSpans() + } + + const nonEmpty = spans.filter(x => !x.empty) + if (spans.length > 0) { + writer.write(nonEmpty.map(getSpanText).join('\n') + '\n') + spans.length = 0 + } + + perfTime += performance.now() - p + + getLogger().debug(`Time spent on UI: ${Math.floor(perfTime * 100) / 100}ms (${frames} frames)`) + } + + function getRow(pos: number) { + + } + + return { write: (text: string) => write(text, true), writeLine, createRow, createFooter, dispose } + } + + function cancelPending(data: ViewData) { + while (data.pendingWrites.length > 0) { + writer.cancel(data.pendingWrites.shift()!) + } + } + + function renderCurrentView() { + const name = getCurrentView() + if (!name) { + writer.clearScreen() + return + } + + const data = views.get(name)! + cancelPending(data) + renderView(data) + } + + function hideView(name: string) { + const index = viewStack.indexOf(name) + if (index === -1) { + return + } + + swap(viewStack, index, viewStack.length - 1) + viewStack.pop() + + if (index === viewStack.length) { + cancelPending(views.get(name)!) + renderCurrentView() + } + } + + let didSetup = false + function swapView(name: string) { + const index = viewStack.indexOf(name) + if (index === -1) { + if (viewStack.length === 0 && !didSetup) { + writer.setupScreen() + didSetup = true + } + viewStack.push(name) + } else { + swap(viewStack, index, viewStack.length - 1) + if (index !== viewStack.length - 1) { + cancelPending(views.get(viewStack[index])!) + } + } + } + + function getCurrentView(): string | undefined { + return viewStack[viewStack.length - 1] + } + + function bound(val: number, max: number, min = 0) { + return Math.max(min, Math.min(val, max)) + } + + function moveCursor(name: string, dx: number, dy: number) { + const data = views.get(name) + if (!data) { + return + } + + const maxRows = (process.stdout.rows - (data.fullScreen ? 0 : 1)) + const column = bound(data.cursorPosition.column + dx, process.stdout.columns - 1) + + const row = bound(data.cursorPosition.row + dy, Math.min( + data.rows.length - 1, + maxRows - 1, + )) + + const scroll = dy < 0 + ? Math.min(0, data.cursorPosition.row + dy) + : Math.max(0, (data.cursorPosition.row + dy) - (maxRows - 1)) + + const currentRow = bound(data.currentRow + scroll, data.rows.length - maxRows) + + const updated = { + ...data, + currentRow, + cursorPosition: { column, row } + } + + views.set(name, updated) + if (getCurrentView() === name) { + cancelPending(data) + renderView(updated) + } + } + + function createView(name: string, opt?: { hideCursor?: boolean }) { + views.set(name, { + rows: [], + currentRow: 0, + pendingWrites: [], + cursorPosition: { row: 0, column: 0 }, + }) + + function show() { + if (getCurrentView() === name) { + return + } + + if (opt?.hideCursor) { + writer.hideCursor() + } + + swapView(name) + renderCurrentView() + } + + function hide() { + hideView(name) + + if (opt?.hideCursor) { + writer.showCursor() + } + } + + function writeLine(line: string) { + const data = views.get(name)! + const currentRow = data.currentRow + 1 + data.rows.push(line) + const updated = { + ...data, + currentRow, + cursorPosition: { + column: 0, + row: Math.min(currentRow, process.stdout.rows) + } + } + + views.set(name, updated) + + if (getCurrentView() === name) { + if (updated.cursorPosition.row === process.stderr.rows) { + cancelPending(updated) + data.pendingWrites.push(writer.writeRows(updated.rows)) + } else { + data.pendingWrites.push(writer.write(line + '\n')) + } + } + } + + function setRows(rows: string[]) { + const data = views.get(name)! + const updated = { + ...data, + rows, + } + views.set(name, updated) + + if (getCurrentView() === name) { + cancelPending(data) + renderView(updated) + } + } + + function dispose() { + hide() + views.delete(name) + + return writer.flush() + } + + return { show, hide, writeLine, setRows, dispose, moveCursor: (dx: number, dy: number) => moveCursor(name, dx, dy) } + } + + writer.tty?.onKeyPress(ev => { + // const currentView = getCurrentView() + // if (currentView) { + // if (ev.key === ControlKey.UpArrow) { + // moveCursor(currentView, 0, -1) + // } else if (ev.key === ControlKey.DownArrow) { + // moveCursor(currentView, 0, 1) + // } + // } + }) + + function _createOverlayableView(): ReturnType { + if (process.stdout.isTTY) { + return createOverlayableView() + } + + const write = (msg: string) => process.stdout.write(stripAnsi(msg)) + + function createDisplayRow(text?: string): DisplayRow { + let currentText = text + + return { + update: text => { currentText = text }, + release: text => { + currentText = text ?? currentText + write(`${currentText}\n`) + }, + destroy: () => {}, + } + } + + return { + write, + writeLine: msg => msg ? write(`${msg}\n`) : write('\n'), + createFooter: text => createDisplayRow(text), + createRow: (text) => createDisplayRow(text), + dispose: async () => {}, + } + } + + const getOverlayedView = memoize(_createOverlayableView) + + async function releaseTty(closeStdout = false) { + if (getOverlayedView.cached) { + const v = getOverlayedView() + await v.dispose() + } + + writer.showCursor() + if (closeStdout) { + await writer.end() + } else { + await writer.flush() + } + writer.tty?.dispose() + } + + async function dispose() { + if (!process.stdout.isTTY) { + return + } + + await releaseTty(true) + } + + writer.tty?.onSignal(ev => { + if (ev.signal === 'SIGINT') { + process.emit('SIGINT') + } + }) + + return { createView, getOverlayedView, dispose, releaseTty } +} + +export interface TreeItem { + readonly id: string + readonly children: TreeItem[] + label: string + visible: boolean + sortOrder: number +} + +export function createTreeView(name: string, display = getDisplay()) { + const view = display.createView(name, { hideCursor: true }) + const items = new Map() + const roots: TreeItem[] = [] + + function renderItem(item: TreeItem, depth = 0): string[] { + const rows: string[] = [] + if (!item.visible) { + return rows + } + + rows.push(`${' '.repeat(depth)}${item.label}`) + + for (const child of item.children.sort((a, b) => a && b ? a.sortOrder - b.sortOrder : 0)) { + // Child could have been deleted + if (child) { + rows.push(...renderItem(child, depth + 1)) + } + } + + return rows + } + + function getRows() { + const rows: string[] = [] + for (const item of roots.sort((a, b) => a.sortOrder - b.sortOrder)) { + rows.push(...renderItem(item)) + } + + return rows + } + + function render() { + view.setRows(getRows()) + } + + function createItem(id: string, label = ''): TreeItem { + if (items.has(id)) { + throw new Error(`Item with id already exists: ${id}`) + } + + const children = new Proxy([] as TreeItem['children'], { + set: (arr, p, val, recv) => { + if (p === 'length' && arr.length !== val) { + render() + + return Reflect.set(arr, p, val, recv) + } + + const index = Number(p) + if (!isNaN(index) && arr[index] !== val) { + render() + } + + return Reflect.set(arr, p, val, recv) + }, + deleteProperty: (arr, p) => { + const index = Number(p) + if (!isNaN(index) && arr[index]) { + render() + } + + return Reflect.deleteProperty(arr, p) + }, + }) + + let visible = true + let sortOrder = 0 + + const item: TreeItem = { + id, + children, + + get label() { + return label + }, + + set label(val: string) { + if (val !== label) { + label = val + render() + } + }, + + get visible() { + return visible + }, + + set visible(val: boolean) { + if (val !== visible) { + visible = val + render() + } + }, + + get sortOrder() { + return sortOrder + }, + + set sortOrder(val: number) { + if (val !== sortOrder) { + sortOrder = val + render() + } + }, + } + + items.set(id, item) + + return item + } + + function addItem(item: TreeItem): void { + roots.push(item) + render() + } + + function _removeItem(arr: TreeItem[], item: TreeItem): boolean { + let didRemove = false + const index = arr.indexOf(item) + if (index !== -1) { + arr.splice(index, 1) + didRemove = true + } + + for (const c of arr) { + if (c.children.length > 0) { + didRemove ||= _removeItem(c.children, item) + } + } + + return didRemove + } + + function removeItem(item: TreeItem): void { + const didRemove = _removeItem(roots, item) + if (didRemove) { + items.delete(item.id) + render() + } + } + + function clearItems() { + items.clear() + roots.length = 0 + render() + } + + return { + addItem, + removeItem, + createItem, + clearItems, + getRows, // Kind of a leaky abstraction but eh + show: () => view.show(), + hide: () => view.hide(), + dispose: () => view.dispose(), + } +} + +// This is preferred over `console.log` because we want to change the output depending on where the tool is ran +export function print(msg: string) { + if (!process.stdout.isTTY) { + return process.stdout.write(stripAnsi(msg)) + } + + // XXX: not a good impl. + const view = getDisplay().getOverlayedView() + if (msg.endsWith('\n')) { + const lines = msg.slice(0, -1).split('\n') + for (const l of lines) { + view.writeLine(l) + } + } else { + view.write(msg) + } +} + +// Why not call this `println`? Because not everyone knows `ln` === `line` +export function printLine(msg: string = '', ...args: any[]) { + return print([msg, ...args].map(String).join(' ') + '\n') +} + +export function printJson(data: any) { + return print(JSON.stringify(data, undefined, 4) + '\n') +} + +export class RenderableError extends Error { + public constructor(message: string, private readonly renderFn: () => Promise | void) { + super(message) + } + + public render() { + return this.renderFn() + } +} + +export interface Spinner { + readonly frames: string[] + readonly rotationsPerSecond: number +} + +const brailleSpinner: Spinner = { + rotationsPerSecond: 1/2, + + // Braille grid + // 1 4 + // 2 5 + // 3 6 + // 7 8 + + frames: [ + '\u2806', // 23 + '\u2807', // 123 + '\u2803', // 12 + '\u280b', // 124 + '\u2809', // 14 + '\u2819', // 145 + '\u2818', // 45 + '\u2838', // 456 + '\u2830', // 56 + '\u2834', // 356 + '\u2824', // 36 + '\u2826', // 236 + + // MacOS system font doesn't show the bottom 2 dots if they're empty + + // '\u28b0', // 568 + // '\u28a0', // 68 + // '\u28e0', // 678 + // '\u28c0', // 78 + // '\u28c4', // 378 + // '\u2844', // 37 + // '\u2846', // 237 + ] +} + +const ellipsisSpinner: Spinner = { + frames: ['', '.', '..', '...'], + rotationsPerSecond: 1/4, +} + +export function getSpinnerFrame(spinner: Spinner, duration: number) { + const framesPerSecond = spinner.frames.length * spinner.rotationsPerSecond + const frame = Math.floor((duration / 1000) * framesPerSecond) + + return spinner.frames[frame % spinner.frames.length] +} + +// circle spinner ['◴', '◷', '◶', '◵'] +// bar spinner +// [ +// '—', // 1 em dash (U+2014) +// '\u27cb', // ⟋ +// '|', +// '\u27cb' // ⟍ +// ] + +const emtpySpinner: Spinner = { + frames: [''], + rotationsPerSecond: 0, +} + +export const spinners = { + empty: emtpySpinner, + braille: brailleSpinner, + ellipsis: ellipsisSpinner, +} satisfies Record + +export function formatDuration(ms: number) { + if (ms >= 100000) { + // Show no decimal places + // 521852 -> 522s + return `${Math.round(ms / 1000)}s` + } + + if (ms >= 10000) { + // Show 1 decimal place + // 18472 -> 18.5s + return `${Math.round(ms / 100) / 10}s` + } + + if (ms >= 1000) { + // Show 2 decimal places + // 1374 -> 1.37s + return `${Math.round(ms / 10) / 100}s` + } + + return `${ms}ms` +} + +export function renderDuration(duration?: number) { + return duration ? dim(` [${formatDuration(duration)}]`) : '' +} + +export function format(...args: any[]) { + return nodeUtil.formatWithOptions({ colors: process.stdout.isTTY }, ...args) +} \ No newline at end of file diff --git a/src/cli/updater.ts b/src/cli/updater.ts new file mode 100644 index 0000000..95b2a44 --- /dev/null +++ b/src/cli/updater.ts @@ -0,0 +1,5 @@ +// TODO: +// 1. Automatically check for updates (in the background, forked proc) +// 2. Auto-update is ok for the global install, for projects users should be aware that we're updating +// * Initial release won't have per-project installs +// 3. This can fetch GitHub releases directly. No need for our own service. diff --git a/src/cli/views/compile.ts b/src/cli/views/compile.ts new file mode 100644 index 0000000..d32a137 --- /dev/null +++ b/src/cli/views/compile.ts @@ -0,0 +1,364 @@ +import * as nodeUtil from 'node:util' +import { bold, colorize, dim, getDisplay, renderDuration } from '../ui' +import { CompilerOptions } from '../../compiler/host' +import { ResolvedProgramConfig } from '../../compiler/config' +import { getLogger } from '../..' +import { getPreviousDeploymentProgramHash, getTemplateWithHashes, readState } from '../../artifacts' +import { TfJson } from '../../runtime/modules/terraform' +import { getBuildTarget } from '../../execution' +import { getWorkingDir } from '../../workspaces' +import { SymbolNode, createMergedGraph, createSymbolGraphFromTemplate } from '../../refactoring' +import { renderBetterSymbolName } from './deploy' +import { TfState } from '../../deploy/state' +import { renderCmdSuggestion } from '../commands' + + +// Useful events +// * Program/process/project resolution (if not obvious) +// * Config resolution (esp. `deployTarget`) +// * Completion per-file +// * Start/end of synthesis +// * User logs during synthesis + +// Potential (future) sources of config, roughly ordered highest precedence first: +// * command line +// * environment variables +// * `package.json` / `tsconfig.json` +// * `~/.synapse/config.json` (per-user; global) +// * Per-project config (might be a file or possibly remote) +// * Organization (?) +// * Hard-coded defaults +// +// A good way to determine what precedence to give a config source is to think about +// how specific it is to a given command invocation. Command line args are for only _that_ +// invocation and so it has the highest weight. But hard-coded defaults can apply to literally +// every single command invocation by every single user, so it should have the lowest. The broader +// the scope, the lower the precedence. +// +// Project/organization config could potentially have the highest precedence if the setting is "locked" +// +// Other tenets: +// * Don't give devs the chance to say "but it works on my machine" + +interface ConfigDelta { + input: T | undefined + resolved: T | undefined // TODO: show where we got this value + clobbered?: boolean // Clobbered input with higher precedence === bug + + // TODO: show other clobbers/deltas somehow (there may be more multiple sources of config) + // I think the easiest way would be to gather all config sources into an array and sort by + // precedence. This array could then be used to create per-input deltas on-demand. + +} + +// This is intentionally hand-written to include only the most relevant things +interface ConfigDiff { + environmentName?: ConfigDelta + deployTarget?: ConfigDelta +} + +function getDelta(input: T | undefined, resolved: T | undefined): ConfigDelta | undefined { + if (input === resolved) { + return + } + + return { + input, + resolved, + clobbered: input !== undefined && input !== resolved, + } +} + +// When should we show a resolved input? +// * Potentially mon-obvious input sources e.g. environment variables, hard-coded defaults, etc. +// * Derived and/or inferred values +// * When one implicit input overrides another (Synapse config vs. `tsconfig.json`) + +function diffConfig( + config: ResolvedProgramConfig, + inputOptions: CompilerOptions = {} +): ConfigDiff { + const diff: ConfigDiff = {} + diff.deployTarget = getDelta(inputOptions.deployTarget, config.csc.deployTarget) + + for (const [k, v] of Object.entries(diff)) { + if (v === undefined) { + delete diff[k as keyof ConfigDiff] + } + } + + return diff +} + +function mapOptionName(name: string) { + switch (name) { + case 'deployTarget': + return 'target' + case 'environmentName': + return 'environment' + } + + return name +} + +const showResourceSummary = false + +interface CompileSummaryOpt { + showResourceSummary?: boolean + showSuggestions?: boolean +} + +// `inputOptions` is used for diffing the resolved config +export function createCompileView(inputOptions?: CompilerOptions & { hideLogs?: boolean }) { + const view = getDisplay().getOverlayedView() + const header = view.createRow('Compiling...') + let headerText = 'Compiling...' + let resolvedConfigText: string | undefined + let duration: number + + function updateHeader() { + const h = `${headerText}${renderDuration(duration)}` + const newText = resolvedConfigText ? `${h} ${resolvedConfigText}` : h + header.update(newText) + } + + function setHeaderText(text: string) { + headerText = text + updateHeader() + } + + function setConfigText(text: string) { + resolvedConfigText = text + updateHeader() + } + + const startTime = Date.now() + + // The config may undergo multiple resolutions + // But individual values must never change after being set + // Anything that uses a particular key must use the resolved form + getLogger().onResolveConfig(ev => { + const diff = diffConfig(ev.config, inputOptions) + const entries = Object.entries(diff) as [string, ConfigDelta][] + if (entries.length === 0) { + return + } + + function renderEntry(k: string, v: ConfigDelta) { + return `${mapOptionName(k)}: ${bold(colorize(v.clobbered ? 'red' : 'blue', v.resolved ?? v.input))}` + } + + const resolved = entries.filter(([k, v]) => v.resolved !== undefined) + const text = dim(`(${resolved.map(([k, v]) => renderEntry(k, v))}${dim(')')}`) + setConfigText(text) + }) + + function done() { + duration = Date.now() - startTime + // Maybe show # of files compiled + setHeaderText(colorize('green', 'Done!')) + header.release() + } + + getLogger().onCompile(ev => { + done() + }) + + let isFirstSynthLog = true + getLogger().onSynthLog(ev => { + if (inputOptions?.hideLogs) { + getLogger().log(``, ...ev.args) + return + } + + if (isFirstSynthLog) { + isFirstSynthLog = false + view.writeLine() + view.writeLine(`Synthesis logs:`) + } + + const formatted = `${dim(`[${ev.level}]`)} ${nodeUtil.format(...ev.args)}` // TODO: show source file + line # + col # + const lines = formatted.split('\n') + for (const l of lines) { + view.writeLine(` ${l}`) + } + }) + + return { showSimplePlanSummary, done } +} + +type PreviousData = Awaited> + +export async function getPreviousDeploymentData() { + const procId = getBuildTarget()?.deploymentId + if (!procId) { + return + } + + const programHash = await getPreviousDeploymentProgramHash() + const [state, oldTemplate] = await Promise.all([ + readState(), + programHash ? getTemplateWithHashes(programHash) : undefined, + ]) + + return { + state, + programHash, + oldTemplate, + } +} + +// async function getStatelessTerraformSession(template: TfJson) { +// return startStatelessTerraformSession(template, await getTerraformPath()) +// } + +// // This diff is "fast" because we make no remote calls +// // +// // We're only diffing the inputs. We cannot know the change in program +// // state (output) without actually executing the program. +// async function getFastDiff(current: TemplateWithHashes, previous: PreviousData) { +// const session = await getStatelessTerraformSession(current.template) + +// } + + +// TODO: we could diff the new template with the last deployed template, if available +function showSimplePlanSummary(template: TfJson, target: string, entrypoints: string[], previousData?: PreviousData) { + const printLine = getDisplay().getOverlayedView().writeLine + + const workingDir = getWorkingDir() + const graph = createSymbolGraphFromTemplate(template) + const oldGraph = previousData?.oldTemplate ? createSymbolGraphFromTemplate(previousData?.oldTemplate.template) : undefined + const mergedGraph = oldGraph ? createMergedGraph(graph, oldGraph) : undefined + + const sorted = graph.getSymbols().sort((a, b) => { + if (a.value.fileName !== b.value.fileName) { + return 0 + } + + if (a.value.line === b.value.line) { + return a.value.column - b.value.column + } + + return a.value.line - b.value.line + }) + + function getResourceCounts(sym: SymbolNode['value']) { + const byType: Record = {} + for (const r of sym.resources) { + if (r.subtype === 'Closure' && r.name.endsWith('--definition')) continue + if (r.subtype === 'Closure' && sym.name === 'describe') continue + + const ty = graph.getResourceType(`${r.type}.${r.name}`) + if (ty.kind === 'custom') { + byType[ty.name] = (byType[ty.name] ?? 0) + 1 + } else if (r.subtype === 'Closure') { + byType[''] = (byType[''] ?? 0) + 1 + } else if (!r.subtype) { + byType[r.type] = (byType[r.type] ?? 0) + 1 + } + } + + return Object.entries(byType).sort((a, b) => a[0].localeCompare(b[0])) + } + + const counts = new Map>() + for (const s of sorted) { + const c = getResourceCounts(s.value) + if (c.length > 0) { + counts.set(s, c) + } + } + + let hasTests = false + for (const s of sorted) { + for (const r of s.value.resources) { + const testInfo = graph.getTestInfo(`${r.type}.${r.name}`) + if (testInfo?.testSuiteId !== undefined) { + hasTests = true + } + } + } + + if (counts.size === 0) { + return printLine('No resources found.') + } + + printLine() + if (showResourceSummary) { + const header = '---------------- Resource Summary ----------------' + + function summarizeSymbol(sym: SymbolNode['value'], entries: ReturnType) { + const symName = renderBetterSymbolName(sym, workingDir, header.length) + printLine(symName) + for (const [k, v] of entries) { + if (k === '') continue + // XXX: big hack to hide custom resource nodes + if (v === 1 && sym.name.endsWith(`new ${k}`)) continue + + const suffix = v > 1 ? ` (x${v})` : '' + printLine(` |_ ${k}${suffix}`) + } + } + + printLine(header) + + for (const [s, c] of counts) { + summarizeSymbol(s.value, c) + } + + printLine() + } + + // entrypoint && !entrypoint.startsWith('--') ? `${cliName} test ${entrypoint}` : + const isUpdate = !!previousData?.state && previousData.state.resources.length > 0 + const deployCmd = renderCmdSuggestion('deploy') + if (hasTests) { + const testCmd = renderCmdSuggestion('test') + printLine(`Commands you can use next:`) + printLine(` ${colorize('cyan', deployCmd)} to ${isUpdate ? 'update' : 'start'} your application`) + printLine(` ${colorize('cyan', testCmd)} to ${isUpdate ? 'update' : 'start'} and test your application`) + } else { + printLine(`Run ${colorize('cyan', deployCmd)} to ${isUpdate ? 'update' : 'start'} your application`) + } + + printLine() + + const previousTarget = previousData?.oldTemplate?.template['//']?.deployTarget + if (previousTarget && previousTarget !== target) { + printLine(colorize('yellow', `Previous deployment used a different target: ${previousTarget}`)) + } +} + +interface ShowWhatICanDoNextProps { + entrypoint?: string + deployTarget?: string + graph?: ReturnType + state?: TfState +} + +// AKA "clippyMode" +function showWhatICanDoNext(command: 'compile' | 'deploy', props?: ShowWhatICanDoNextProps) { + // After compile: + // * `deploy` + // * `test` (only if there are test resources) + // * `plan` (only if there is an existing process) + // * `run` (only if we found a `main` function _and_ it doesn't need deployment) + + // After deploy: + // * `destroy` + // * `show` + // * `test` (only if there are test resources) + // * `repl` (need to make it obvious that you can use any file in your program as an entrypoint) + + const thingsToDo: string[] = [] + + switch (command) { + case 'compile': { + + } + } + + // If the # of next steps if 1, show it on one line (?) + // Also add something to say how to disable these suggestions +} \ No newline at end of file diff --git a/src/cli/views/deploy.ts b/src/cli/views/deploy.ts new file mode 100644 index 0000000..0774fd2 --- /dev/null +++ b/src/cli/views/deploy.ts @@ -0,0 +1,576 @@ +import * as path from 'node:path' +import { getLogger } from '../..' +import { ParsedPlan, getChangeType, mapResource } from '../../deploy/deployment' +import { DeployEvent, DeploySummaryEvent, FailedDeployEvent } from '../../logging' +import { SymbolNode, SymbolGraph, renderSymbol, MergedGraph, renderSymbolLocation } from '../../refactoring' +import { Color, colorize, createTreeView, format, getDisplay, getSpinnerFrame, printLine, Spinner, spinners, stripAnsi, TreeItem } from '../ui' +import { keyedMemoize } from '../../utils' +import { getWorkingDir } from '../../workspaces' +import { resourceIdSymbol } from '../../deploy/server' + +interface ResourceInfo { + internal?: boolean + status?: DeployEvent['status'] + action: DeployEvent['action'] + resourceType: ReturnType + + // Only relevant for `replace` + destroyed?: boolean +} + +interface SymbolState { + status: DeployEvent['status'] + action: DeployEvent['action'] + resources: Record + startTime?: Date +} + + +function getBetterName(sym: SymbolNode['value']) { + const parts = sym.name.split(' = ') + if (parts.length === 1) { + return { name: parts[0] } + } + + const ident = parts[0] + const rem = parts[1] + if (rem.startsWith('new ')) { + return { name: ident, type: rem.slice(4) } + } + + return { name: ident } +} + +export function renderBetterSymbolName( + sym: SymbolNode['value'], + workingDir: string, + headerLength: number, + icon = '*', + status = '' +) { + + const parts = getBetterName(sym) + const pSym = { + ...sym, + name: `${parts.name}${parts.type ? ` <${parts.type}>` : ''}`, + fileName: path.relative(workingDir, sym.fileName), + } + + const loc = renderSymbolLocation(pSym, true) + const s = `${icon} ${renderSymbol(pSym, false)}${status}` + const padded = s.padEnd(headerLength - loc.length, ' ') + + return `${padded}${loc}` +} + +function getStatusIcon(status: SymbolState['status'], spinner: Spinner, duration: number) { + switch (status) { + case 'complete': + return colorize('green', '\u2713') // ✓ + case 'failed': + return colorize('red', '\u2717') // ✗ + + default: + return getSpinnerFrame(spinner, duration) + } +} + +function isOnlyUpdatingDefs(state: SymbolState) { + for (const k of Object.keys(state.resources)) { + // XXX + if (!k.endsWith('--definition')) { + return false + } + } + + return true +} + +export function printSymbolTable(symbols: Iterable<[sym: SymbolNode, state: SymbolState]>, showLocation = true, maxWidth = 80) { + const texts = new Map() + for (const [k, v] of symbols) { + if (isOnlyUpdatingDefs(v)) continue + + const text = renderSymbolWithState(k.value, v, undefined, spinners.empty) + texts.set(k, text) + } + + if (texts.size === 0) { + return + } + + // const headerSize = Math.min(process.stdout.columns, maxWidth) + const largestWidth = [...texts.values()].map(stripAnsi).sort((a, b) => b.length - a.length)[0] + const minGap = 2 + const padding = largestWidth.length + minGap + + for (const [k, v] of symbols) { + if (isOnlyUpdatingDefs(v)) continue + + const relPath = path.relative(getWorkingDir(), k.value.fileName) + const left = renderSymbolWithState(k.value, v, undefined, spinners.empty) + const right = renderSymbolLocation({ ...k.value, fileName: relPath }, true) + const leftWidth = stripAnsi(left).length + //const padding = headerSize - leftWidth + printLine(`${left}${' '.repeat(padding - leftWidth)}${colorize('gray', right)}`) + } +} + +export function renderSymbolWithState( + sym: SymbolNode['value'], + state: SymbolState, + workingDir = getWorkingDir(), + spinner = spinners.braille +) { + const actionColor = getColor(state.action) + const icon = getIcon(state.action) + // const status = state.status !== 'pending' ? ` (${state.status})` : '' + const duration = state.startTime ? Date.now() - state.startTime.getTime() : undefined + const seconds = duration ? Math.floor(duration / 1000) : 0 + const durationText = !seconds ? '' : ` (${seconds}s)` + + const parts = getBetterName(sym) + + const status = getStatusIcon(state.status, spinner, duration ?? 0) + const symType = parts.type ? ` <${parts.type}>` : '' + + const failed = state.status === 'failed' ? colorize('brightRed', ' [failed]') : '' + + const loc = '' // renderSymbolLocation(pSym, true) + const padding = '' // ' '.repeat(Math.max(headerLength - lhsSize - loc.length - 1, 0)) + + const details = colorize('gray', `${symType}${durationText}${failed}${padding}${loc}`) + const symWithIcon = colorize(actionColor, `${icon} ${parts.name}`) + + return `${status} ${symWithIcon}${details}` +} + +interface DeploySummary { + +} + +export async function createDeployView(graph: MergedGraph, isDestroy?: boolean) { + const view = getDisplay().getOverlayedView() + const headerLength = Math.min(process.stdout.columns, 80) + const opName = isDestroy ? 'destroy' : 'deploy' + const header = view.createRow(`Planning ${opName}...`) + const errors: [resource: string, reason: (string | Error)][] = [] + const skipped = new Set() + + function _getRow(id: string) { + return view.createRow() + } + + const getRow = keyedMemoize(_getRow) + const timers = new Map() + + // TODO: continously update `duration` until the symbol is complete + // TODO: for multiple file deployments we should show status per-file rather than per-symbol + + getLogger().onPlan(ev => { + const symbolStates = extractSymbolInfoFromPlan(graph, ev.plan) + if (symbolStates.size === 0) { + if (isDestroy) { + header.release(colorize('green', 'Nothing to destroy!')) + } else { + header.release(colorize('green', 'No changes needed')) + } + + return + } + + function updateSymbolState(ev: DeployEvent | FailedDeployEvent) { + if (!graph.hasSymbol(ev.resource)) { + return + } + + const symbol = graph.getSymbol(ev.resource) + const state = symbolStates.get(symbol) + if (!state) { + return + } + + const resource = state.resources[ev.resource] + // TODO: figure out why this is `undefined` when destroying test resources + if (!resource) { + return + } + + const action = resource.action + if (action === 'noop') { + resource.status = 'complete' + } else if (action === 'replace' && ev.status === 'complete') { + if (!resource.destroyed) { + resource.destroyed = true + } else { + resource.status = 'complete' + } + } else { + state.resources[ev.resource] = { ...resource, ...ev, action } + } + + if (ev.status === 'applying') { + state.startTime ??= new Date() + } + + const statuses = Object.values(state.resources).map(ev => ev.status) + if (statuses.every(s => s === 'complete')) { + state.status = 'complete' + } else if (statuses.some(s => s === 'failed')) { + state.status = 'failed' + } else if (statuses.some(s => s !== 'pending')) { + state.status = 'applying' + } + + return { + symbol, + state, + } + } + + // Internal resources are considered "boring" because: + // 1. No remote calls are made + // 2. Any state changes only exist locally + // 3. Failures are not usually the user's fault + // + // So we generally do not want to show them + const boringSymbols = new Set() + + let total = 0 + for (const [k, v] of symbolStates) { + let hasInteresting = false + for (const r of Object.values(v.resources)) { + if (!r.internal) { + total += 1 + hasInteresting= true + } + } + + if (!hasInteresting) { + total += 1 + boringSymbols.add(k.value.id) + } + } + + let completed = 0 + let failed = 0 + const action = isDestroy ? 'Destroying' : 'Deploying' + function updateHeader() { + const failedMsg = failed ? colorize('red', ` ${failed} ${failed > 1 ? 'failures' : 'failure'}`) : '' + header.update(`${action} (${completed}/${total})${failedMsg}`) + } + + updateHeader() + + function addRenderInterval(symbol: SymbolNode, state: SymbolState) { + if (timers.has(symbol.value.id)) { + return + } + + const r = getRow(`${symbol.value.id}`) + const t = +setInterval(() => { + const s = renderSymbolWithState( + symbol.value, + state, + getWorkingDir(), + ) + + r.update(s) + }, 250) + timers.set(symbol.value.id, t) + } + + function clearRenderInterval(symbol: SymbolNode) { + clearInterval(timers.get(symbol.value.id)) + } + + function onDeployEvent(ev: DeployEvent | FailedDeployEvent) { + const res = updateSymbolState({ ...ev, state: mapResource(ev.state) }) + if (!res) { + return + } + + const r = getRow(`${res.symbol.value.id}`) + const state = res.state + addRenderInterval(res.symbol, res.state) + + const s = renderSymbolWithState( + res.symbol.value, + state, + getWorkingDir(), + ) + + if (state.resources[ev.resource].status === 'complete') { + if (!state.resources[ev.resource].internal) { + completed += 1 + updateHeader() + } + } else if (state.resources[ev.resource].status === 'failed') { + if (!state.resources[ev.resource].internal) { + failed += 1 + updateHeader() + } + } + + if (state.status === 'complete') { + if (boringSymbols.has(res.symbol.value.id)) { + completed += 1 + } + + clearRenderInterval(res.symbol) + r.release(s, 2500) + updateHeader() + } else if (state.status === 'failed') { + if (boringSymbols.has(res.symbol.value.id)) { + failed += 1 + } + + clearRenderInterval(res.symbol) + r.release(s) + } else { + r.update(s) + } + } + + getLogger().onDeploy(ev => { + // if (ev.status === 'failed') { + // errors.push([ev.resource, (ev as any).reason]) + // } + if (ev.action === 'noop') { + skipped.add(ev.resource) + } + + try { + onDeployEvent(ev) + } catch (e) { + getLogger().error(e) + } + }) + }) + + const resourceLogs: Record = {} + getLogger().onDeployLog(ev => { + const arr = resourceLogs[ev.resource] ??= [] + arr.push(require('node:util').format(...ev.args)) + }) + + function dispose() { + header.release() + for (const [id] of getRow.keys()) { + getRow(id).release() + } + for (const v of timers.values()) { + clearInterval(v) + } + + if (errors.length > 0) { + view.writeLine() + view.writeLine('Errors:') + + // TODO: convert resource names to symbols + for (const [r, e] of errors) { + if (typeof e === 'string') { + view.writeLine(`[${r}]: ${e}`) + } else { + printLine(`[${r}]: ${format(e)}`) + } + } + } + + if (skipped.size > 0) { + getLogger().log(`Skipped:`, skipped) + } + + const l = Object.entries(resourceLogs) + if (l.length > 0) { + view.writeLine() + view.writeLine('Resource logs:') + // TODO: convert resource names to symbols + for (const [k, v] of l) { + for (const z of v) { + view.writeLine(`[${k}]: ${z}`) + } + } + } + } + + function formatError(err: any) { + const r = err[resourceIdSymbol] + + if (r && graph.hasSymbol(r)) { + delete err[resourceIdSymbol] + + const callsites = graph.getCallsites(r) + if (!callsites) { + return format(err) + } + + // These callsites are already source-mapped, filenames are relative though + const s = callsites.map(c => ` at ${c.name} (${c.fileName}:${c.line + 1}:${c.column + 1})`).join('\n') + err.stack += '\n' + s + } + + return format(err) + } + + return { dispose, formatError } +} + +function summarizePlan(graph: MergedGraph, plan: ParsedPlan): DeployEvent['action'] | 'no-op' { + let action: DeployEvent['action'] | 'no-op' = 'no-op' + for (const [k, v] of Object.entries(plan)) { + // if (graph.isInternalResource(k) || k.startsWith('data.')) { + // continue + // } + + if (k.startsWith('data.')) { + continue + } + + const change = getChangeType(v.change) + + switch (action) { + case 'no-op': + action = change + break + case 'read': + continue + + default: + if (change !== action) { + return 'update' + } + break + } + } + + return action +} + +export function groupSymbolInfoByFile(info: Map): Record { + const groups: Record = {} + for (const [k, v] of info) { + const g = groups[k.value.fileName] ??= [] + g.push([k, v]) + } + + return groups +} + +export function extractSymbolInfoFromPlan(graph: MergedGraph, plan: ParsedPlan) { + const symbolStates = new Map() + const plans = new Map() + for (const [k, v] of Object.entries(plan)) { + if (!graph.hasSymbol(k)) { + continue + } + + const symbol = graph.getSymbol(k) + if (!plans.has(symbol)) { + plans.set(symbol, {}) + } + + const plan = plans.get(symbol)! + plan[k] = v + } + + for (const [k, v] of plans) { + const summary = summarizePlan(graph, v) + if (summary === 'no-op') { + continue + } + + const resources: SymbolState['resources'] = {} + for (const [rId, planned] of Object.entries(v)) { + const action = getChangeType(planned.change) + if (action === 'no-op') { + continue + } + + resources[rId] = { + action, + status: 'pending', + internal: graph.isInternalResource(rId), + resourceType: graph.getResourceType(rId)!, + } + } + + symbolStates.set(k, { + action: summary, + status: 'pending', + resources, + }) + } + + const sorted = [...symbolStates.entries()].sort((a, b) => a[0].value.line - b[0].value.line) + + return new Map(sorted) +} + +function getIcon(action: DeployEvent['action']) { + switch (action) { + case 'create': + return '+' + case 'delete': + return '-' + case 'update': + return '~' + case 'replace': + return '±' + case 'read': + return '^' + } +} + +function getColor(action: DeployEvent['action']): Color { + switch (action) { + case 'create': + return 'green' + case 'delete': + return 'red' + case 'replace': + return 'yellow' + case 'update': + // return 'none' + return 'blue' + case 'noop': + case 'read': + return 'none' + } +} + +export function renderSummary(ev: DeploySummaryEvent) { + if (ev.add === 0 && ev.change === 0 && ev.remove === 0) { + const noChangesMessage = colorize('green', 'No changes needed') + + return noChangesMessage + } + + if (ev.errors && ev.errors.length > 0) { + const deployFailed = colorize('red', 'Deploy failed!') + + return deployFailed // TODO: enumerate errors + } + + const deploySucceeded = colorize('green', 'Deploy succeeded!') + const lines = [ + deploySucceeded, + '', + 'Resource summary:', + ] + + if (ev.add > 0) { + lines.push(` - ${ev.add} created`) + } + if (ev.change > 0) { + lines.push(` - ${ev.change} changed`) + } + if (ev.remove > 0) { + lines.push(` - ${ev.remove} destroyed`) + } + + // One empty line for padding + lines.push('') + + return lines.join('\n') +} diff --git a/src/cli/views/install.ts b/src/cli/views/install.ts new file mode 100644 index 0000000..8f6058b --- /dev/null +++ b/src/cli/views/install.ts @@ -0,0 +1,194 @@ +import { BaseOutputMessage, getLogger } from '../../logging' +import { DepsDiff } from '../../pm/packageJson' +import { compareVersions } from '../../pm/versions' +import { memoize } from '../../utils' +import { colorize, dim, getDisplay, printLine, renderDuration } from '../ui' + +interface InstallLifecycleEventBase extends BaseOutputMessage { + readonly type: 'install-lifecycle' +} + +interface InstallEventStart extends InstallLifecycleEventBase { + readonly phase: 'start' +} + +interface InstallEventResolve extends InstallLifecycleEventBase { + readonly phase: 'resolve' + readonly resolveCount?: number +} + +interface InstallEventDownload extends InstallLifecycleEventBase { + readonly phase: 'download' + readonly packages: string[] +} + +interface InstallEventWrite extends InstallLifecycleEventBase { + readonly phase: 'write' + readonly numPackages: number + // readonly packages: string[] +} + +interface InstallEventEnd extends InstallLifecycleEventBase { + readonly phase: 'end' + readonly summary: InstallSummary +} + +export interface InstallSummary { + readonly rootDeps: Record + + readonly installed?: string[] + readonly changed?: string[] + readonly removed?: string[] + readonly diff?: DepsDiff +} + +export type InstallLifecycleEvent = + | InstallEventStart + | InstallEventResolve + | InstallEventDownload + | InstallEventWrite + | InstallEventEnd + +export interface PackageProgressEvent extends BaseOutputMessage { + readonly type: 'install-package' + readonly phase: 'download' | 'install' + readonly package: string + // readonly totalSizeBytes: number + // readonly bytesDownloaded: number + readonly done?: boolean + readonly cached?: boolean + readonly skipped?: boolean + readonly error?: Error +} + + +export function createInstallView() { + const view = getDisplay().getOverlayedView() + const getHeader = memoize(() => view.createRow('Installing packages...')) + const completed = new Set() + const requested = new Set() + let skipped = 0 + const startTime = Date.now() + + function updateDownloading() { + const suffix = colorize('gray', ` (${completed.size} / ${requested.size})`) + getHeader().update(`Installing...${suffix}`) // Download/install + } + + let summary: InstallSummary + const onInstall = getLogger().onInstall(ev => { + const header = getHeader() + switch (ev.phase) { + case 'start': + header.update('Starting package install...') + break + case 'resolve': + header.update(`Resolving...${ev.resolveCount ? ` ${colorize('gray', `(${ev.resolveCount} packages)`)}` : ''}`) + break + case 'download': + ev.packages.forEach(p => requested.add(p)) + updateDownloading() + break + case 'write': + header.update(`Creating node_modules...`) + break + case 'end': + header.destroy() + getLogger().log(`Total packages installed: ${completed.size - skipped}`) + summary = ev.summary + + // need to show errors + break + } + }) + + const onProgress = getLogger().onPackageProgress(ev => { + if (ev.done) { + completed.add(ev.package) + updateDownloading() + + if (ev.skipped) { + skipped += 1 + } + } + }) + + function summarize() { + if (!summary) { + if (completed.size === 0) { + printLine(colorize('green', 'Done!') + ' No changes needed') + } else { + printLine(colorize('green', 'Done!') + ` ${completed.size} packages installed`) + } + return + } + + const installCount = summary.installed?.length ?? 0 + const removeCount = summary.removed?.length ?? 0 + const changeCount = summary.changed?.length ?? 0 + const total = installCount + removeCount + changeCount + const dur = renderDuration(Date.now() - startTime) + + if (total === 0) { + printLine(colorize('green', 'Done!') + dur + colorize('green', ' No changes needed')) + return + } + + const suffix = ` package${(installCount + removeCount + changeCount) > 1 ? 's' : ''}` + + const addedText = ` Added ${colorize('green', String(installCount))}` + const removedText = ` Removed ${colorize('red', String(removeCount))}` + const changedText = ` Changed ${colorize('blue', String(changeCount))}` + + const arr: string[] = [] + if (installCount) arr.push(addedText) + if (removeCount) arr.push(removedText) + if (changeCount) arr.push(changedText) + + const finalText = arr.map((x, i) => i === 0 ? x : x.toLowerCase()).join(',') + suffix + + + printLine(colorize('green', 'Done!') + dur + `${finalText}`) + + for (const [k, v] of Object.entries(summary.rootDeps)) { + if (v.name?.startsWith('file:/')) continue // XXX: this filters user-installed pkgs, not just internal ones + + const displayName = k === v.name || !v.name + ? `${k}@${v.installedVersion}` + : `${k} - ${v.name}@${v.installedVersion}` + + const latestVersion = v.latestVersion && v.latestVersion !== v.installedVersion + ? dim(` (latest: ${v.latestVersion})`) + : '' + + if (summary.diff?.added && k in summary.diff.added) { + printLine(` ${colorize('green', `+ ${displayName}`)}${latestVersion}`) + } else if (summary.diff?.changed && k in summary.diff.changed) { + const from = summary.diff.changed[k].from.pattern + const to = summary.diff.changed[k].to.pattern + const order = from && to ? compareVersions(from, to) : undefined + const directionText = order + ? colorize(order < 0 ? 'green' : 'red', dim(` [${order < 0 ? 'upgraded' : 'downgraded'}]`)) + : '' + printLine(` ${colorize('blue', `~ ${displayName}`)}${directionText}${latestVersion}`) + } else if (summary.diff?.removed && k in summary.diff.removed) { + printLine(` ${colorize('red', `- ${displayName}`)}`) + } else { + // idk + } + } + } + + function dispose() { + //view.dispose() + onInstall.dispose() + onProgress.dispose() + } + + return { summarize, dispose } +} + diff --git a/src/cli/views/test.ts b/src/cli/views/test.ts new file mode 100644 index 0000000..cd6e5c9 --- /dev/null +++ b/src/cli/views/test.ts @@ -0,0 +1,104 @@ +import { getLogger } from '../..' +import { DeployLogEvent, FailedTestEvent, TestEvent, TestLogEvent } from '../../logging' +import { colorize, format, printLine } from '../ui' +import * as nodeUtil from 'node:util' + +// We should output the filename for each suite/test +export function createTestView() { + const startTimes = new Map() + function getDuration(id: number, endTime: Date) { + const startTime = startTimes.get(id) + if (!startTime) { + return + } + + const dur = endTime.getTime() - startTime.getTime() + + return dur < 5 ? 0 : dur + } + + const indentLevel = new Map() + function getIndent(ev: TestEvent) { + if (ev.parentId === undefined) { + indentLevel.set(ev.id, 0) + + return '' + } + + const parentIndent = indentLevel.get(ev.parentId) ?? -1 + const indent = parentIndent + 1 + indentLevel.set(ev.id, indent) + + return ' '.repeat(indent) + } + + const l = getLogger().onTest(ev => { + // TODO: dynamically show # of tests pending when in tty + if (ev.status === 'pending') { + return + } + + if (ev.status === 'running') { + if (ev.itemType === 'suite') { + printLine(`${getIndent(ev)}- ${ev.name}`) + } + + return startTimes.set(ev.id, ev.timestamp) + } + + if (ev.itemType === 'suite') { + return + } + + const duration = ev.status === 'passed' || ev.status === 'failed' + ? getDuration(ev.id, new Date()) + : undefined + + const durationText = duration ? colorize('gray', ` (${duration}ms)`) : '' + + // We assume that test events come in sequentially + if (ev.status === 'passed') { + printLine(getIndent(ev) + colorize('green', `${ev.name}${durationText}`)) + } else { + printLine(getIndent(ev) + colorize('red', `${ev.name}${durationText}`)) + } + }) + + function showFailures(failures: FailedTestEvent[]) { + for (const ev of failures) { + // XXX: don't show test suite failures caused by a child test failing + if (failures.find(x => x.parentId === ev.id)) { + continue + } + + printLine('\n') + printLine( + colorize('red', `[FAILED] ${ev.name}`), + nodeUtil.formatWithOptions({ colors: process.stdout.isTTY }, ev.reason) + ) + } + } + + const testLogs: (TestLogEvent | DeployLogEvent)[] = [] + + // getLogger().onTestLog(ev => testLogs.push(ev)) + // TODO: tests always emit deploy logs atm + // This is OK because we can get the test id from the resource id + getLogger().onDeployLog(ev => testLogs.push(ev)) + + function dispose() { + l.dispose() + if (testLogs.length > 0) { + printLine() + printLine('Test logs:') + for (const ev of testLogs) { + printLine(` ${format(...ev.args)}`) + } + } + } + + return { + showFailures, + dispose, + } +} \ No newline at end of file diff --git a/src/closures.ts b/src/closures.ts new file mode 100644 index 0000000..95154a5 --- /dev/null +++ b/src/closures.ts @@ -0,0 +1,569 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { Optimizer, createPointerMapper, createSerializerHost, createTranspiler, getNpmDeps, renderFile } from './bundler' + +import type { ExternalValue } from './runtime/modules/serdes' +import { getLogger } from './logging' +import { BuildFsFragment, toFs } from './artifacts' +import { DeploymentContext } from './deploy/server' +import { coerceToPointer, extractPointers, isDataPointer, pointerPrefix } from './build-fs/pointers' +import { getBuildTargetOrThrow, getFs } from './execution' +import { BuildTarget } from './workspaces' +import { loadBuildState } from './deploy/session' +import { Mutable, getHash, makeExecutable, memoize } from './utils' +import { optimizeSerializedData } from './optimizer' +import { which } from './utils/process' + +function replaceGlobals(captured: any, globals: any) { + if (typeof globals !== 'object' || globals === null) { + getLogger().log('Globals was not an object', globals) + + return captured + } + + // Just in case it gets serialized + if (moveableStr in globals) { + globals = globals[moveableStr]['properties'] + } + + function visit(obj: any): any { + if (obj && typeof obj === 'object') { + if (isDataPointer(obj)) { + return obj + } + + for (const [k, v] of Object.entries(obj)) { + if (k === moveableStr && typeof v === 'object' && !!v) { + const desc = v as any + if (desc.valueType === 'reflection' && desc.operations[0].type === 'global' && desc.operations.length >= 2) { + const target = desc.operations[1] + if (target.type === 'get') { + const prop = target.property + const replacement = globals[prop] + if (replacement) { + obj[k] = replacement[moveableStr] + } + } + } + } + + if (obj[k] === v) { + obj[k] = replaceGlobals(v, globals) + } + } + } + + return obj + } + + return visit(captured) +} + +const moveableStr = '@@__moveable__' + +interface BundleOptions { + readonly external?: string[] + readonly destination?: string + readonly bundled?: boolean + readonly platform?: 'node' | 'browser' + readonly immediatelyInvoke?: boolean // This should _only_ be set if bundling a function with no parameters + readonly moduleTarget?: 'esm' | 'cjs' + + readonly publishName?: string + readonly isModule?: boolean + readonly lazyLoad?: string[] + // TODO: combine with `lazyLoad` + readonly lazyLoad2?: string[] + + readonly includeAssets?: boolean + + /** + * @experimental not well tested + */ + readonly useOptimizer?: boolean +} + +// TODO: compiler options should be stored in the template +function getCompilerOptions(workingDirectory: string, outputDirectory?: string): ts.CompilerOptions { + const configFilename = path.join(workingDirectory, 'tsconfig.json') + const config = ts.sys.readFile(configFilename) + if (!config) { + getLogger().log(`No tsconfig.json file found in "${workingDirectory}", using default options`) + + return { + sourceMap: true, + alwaysStrict: true, + rootDir: workingDirectory, + target: ts.ScriptTarget.ES2020, + outDir: outputDirectory ?? workingDirectory, + } + } + + const res = ts.parseConfigFileTextToJson(configFilename, config) + if (!res.config) { + throw new (Error as any)('Bad tsconfig', { cause: res.error }) + } + + const cmd = ts.parseJsonConfigFileContent(res.config, ts.sys, workingDirectory, undefined, configFilename) + + return cmd.options +} + +function createDataTable(captured: any) { + const table: Record = {} + + function visit(val: any): any { + if (Array.isArray(val)) { + return val.map(visit) + } + + if (typeof val !== 'object' || !val) { + return val + } + + if (isDataPointer(val)) { + return val + } + + const result: Record = {} + for (const [k, v] of Object.entries(val)) { + if (k !== moveableStr || typeof v !== 'object' || !v) { + result[k] = visit(v) + continue + } + + const desc = v as any + if (typeof desc['id'] === 'number' || typeof desc['id'] === 'string') { + const id = desc['id'] + table[id] ??= visit(v) + result[k] = { id } + } else { + result[k] = visit(v) + } + } + + return result + } + + return { + table, + captured: visit(captured), + } +} + +export function normalizeSymbolIds(obj: ReturnType) { + if (!isDeduped(obj)) { + return obj + } + + if (isNormalized(obj)) { + return obj + } + + let boundSymbols = 0 + let unboundSymbols = 0 + const symbolMapping = new Map() + + function getId(symbolId: number | string) { + const mapped = symbolMapping.get(symbolId) + if (mapped) { + return mapped + } + + const isBound = typeof symbolId === 'string' && symbolId.startsWith('b:') + const id = isBound ? `b:${boundSymbols}` : `${unboundSymbols}` + if (isBound) { + boundSymbols += 1 + } else { + unboundSymbols += 1 + } + + symbolMapping.set(symbolId, id) + + return id + } + + function visit(val: any): any { + if (Array.isArray(val)) { + return val.map(visit) + } + + if (typeof val !== 'object' || !val) { + return val + } + + if (isDataPointer(val)) { + return val + } + + const result: Record = {} + for (const [k, v] of Object.entries(val)) { + if (k !== moveableStr || typeof v !== 'object' || !v) { + result[k] = visit(v) + continue + } + + const desc = v as any + if (typeof desc['id'] === 'number' || typeof desc['id'] === 'string') { + const id = desc['id'] + const mapped = getId(id) + result[k] = { id: mapped } + } else { + result[k] = visit(v) + } + } + + return result + } + + const table: Record = {} + for (const [k, v] of Object.entries(obj.table)) { + const mapped = getId(k) + if (typeof v === 'object' && !!v && 'id' in v) { + v.id = mapped + if (v.valueType === 'binding') { + if (typeof v.valueType === 'object') { + throw new Error(`Found unexpected object in late binding value: ${JSON.stringify(v.valueType)} [at key ${k}]`) + } + ;(v as Mutable).value = getId(v.value!) + table[mapped] = v as any + continue + } + } + + table[mapped] = visit(v) + } + + return { + ...obj, + __isNormalized: true, + table, + captured: visit(obj.captured), + } +} + +export async function getImportMap(ctx: Pick, table: Record) { + const manifest = ctx.packageManifest + const npmDeps = getNpmDeps(table, manifest) + if (Object.keys(npmDeps.roots).length > 0) { + return ctx.packageService.getImportMap(npmDeps) + } +} + +async function getPackageDependencies(ctx: DeploymentContext, table: Record) { + const manifest = ctx.packageManifest + const npmDeps = getNpmDeps(table, manifest) + if (Object.keys(npmDeps.roots).length > 0) { + return npmDeps + } +} + +const findRuntimeExecutable = memoize(async () => { + const hasNode = await which('node').then(r => true, e => false) + if (hasNode) { + return 'node' + } + return 'synapse' +}) + +function createSeaCode() { + return ` +process.env.SKIP_SEA_MAIN = '1' // XXX: temporary hack for building Synapse + +if (!require('node:v8').startupSnapshot.isBuildingSnapshot()) { + throw new Error("We're building an SEA but we're not building a snapshot") +} +require('node:v8').startupSnapshot.setDeserializeMainFunction(() => { + return main(...process.argv.slice(2)) +}) +`.trim() +} + +export async function bundleExecutable( + bt: BuildTarget, + target: string, + outfile = target, + workingDirectory = bt.workingDirectory, + opt?: BundleOptions & { minifyKeepWhitespace?: boolean; useOptimizer?: boolean; runtimeExecutable?: string; sea?: boolean } +) { + const { mountedFs, resolver, repo } = await loadBuildState(bt) + + const importDecl = opt?.moduleTarget === 'esm' + ? `import { main } from './${path.basename(target)}'` + : `const { main } = require('./${path.basename(target)}')` + + + async function getBanner() { + if (opt?.sea) { + return '' + } + + const runtimeExecutable = opt?.runtimeExecutable ?? await findRuntimeExecutable() + + return `#!/usr/bin/env ${runtimeExecutable}` + } + + const entrypoint = opt?.sea ? createSeaCode() : 'main(...process.argv.slice(2))' + const contents = [ + await getBanner(), + importDecl, + entrypoint, + ].join('\n') + + // XXX: pretty hacky + const emitFs = await repo.getRootBuildFs(`${bt.programId}-emit`) + const optimizer = opt?.useOptimizer ? createOptimizer({ + readDataSync: repo.readDataSync, + writeDataSync: (data) => emitFs.root.writeDataSync(data), + }) : undefined + + const serializerHost = createSerializerHost(emitFs.root, undefined, optimizer) + const transpiler = createTranspiler(mountedFs, resolver, {}) + + const sourceFileName = path.resolve(workingDirectory, target).replace('.js', '.bundled.js') + const res = await transpiler.transpile( + sourceFileName, + contents, + outfile, + { workingDirectory, bundleOptions: { ...opt, serializerHost: serializerHost, bundled: true } } + ) + + await getFs().writeFile(outfile, res.result.contents) + await makeExecutable(outfile) + + const assets = serializerHost.getAssets() + + return { assets, outfile } +} + +export async function bundlePkg( + target: string, + workingDirectory: string, + outfile: string, + opt?: BundleOptions & { minifyKeepWhitespace?: boolean } +) { + const bt = getBuildTargetOrThrow() + const { mountedFs, repo, resolver } = await loadBuildState(bt) + + const transpiler = createTranspiler(mountedFs, resolver, {}) + + // XXX: pretty hacky + const emitFs = await repo.getRootBuildFs(`${bt.programId}-emit`) + const serializerHost = createSerializerHost(emitFs.root) + const sourceFileName = path.resolve(workingDirectory, target) + const res = await transpiler.transpile( + sourceFileName, + await mountedFs.readFile(sourceFileName), + outfile, + { workingDirectory, bundleOptions: { ...opt, serializerHost, bundled: true } } + ) + + await getFs().writeFile(outfile, res.result.contents) + + const assets = serializerHost.getAssets() + + return { assets } +} + +function createOptimizer(fs: { readDataSync: (hash: string) => Uint8Array; writeDataSync: (data: Uint8Array) => string }): Optimizer { + return (table, captured) => { + return optimizeSerializedData(table, captured, (p) => { + const abs = coerceToPointer(p) + const obj = JSON.parse(Buffer.from(fs.readDataSync(abs.hash)).toString()) + if (obj.kind === 'deployed') { + return ts.createSourceFile(abs.hash, '', ts.ScriptTarget.Latest, true) + } + + const data = Buffer.from(obj.runtime, 'base64') + return ts.createSourceFile(abs.hash, data.toString('utf-8'), ts.ScriptTarget.Latest, true) + }, fs.writeDataSync) + } +} + +export async function bundleClosure( + ctx: DeploymentContext, + buildFs: BuildFsFragment, + target: string, // TODO: rename to `source`, make optional + captured: any, + globals: any, + workingDirectory: string, + outputDirectory: string, + opt?: BundleOptions +) { + if (opt?.platform !== 'browser') { + captured = replaceGlobals(captured, globals) + } + + const bundled = opt?.bundled ?? true + const isArtifact = !bundled && !opt?.destination + const extname = opt?.moduleTarget === 'esm' ? '.mjs' : '.cjs' + + const compilerOptions = getCompilerOptions(workingDirectory, outputDirectory) + const outDir = compilerOptions.outDir ?? workingDirectory + const dest = opt?.destination ?? target.replace(/\.(?:t|j)(sx?)$/, '-bundled.j$1') + const outfile = path.resolve(outDir, dest) + + // TODO: normalize all symbol ids, store a mapping in metadata so it can be reversed + // This is somewhat similar to position-independent code + // + // For custom resource handlers it should be ok to map the symbols without storing anything + const data = createDataExport(captured) + + // We have to do this before we render because rendering currently mutates to serialize + const artifacts = Object.values(data.table) + .filter(x => x?.valueType === 'function') + .filter(x => x!.module.startsWith(pointerPrefix)) + .map(x => { + if (isDataPointer(x.module)) { + return x.module + } + + return x!.module.slice(pointerPrefix.length) + }) + + const dependencies = Array.from(new Set(artifacts)) + const packageDependencies = !bundled ? await getPackageDependencies(ctx, data.table) : undefined + if (packageDependencies) { + getLogger().debug(`Found package dependencies for target "${target}"`, Object.values(packageDependencies.packages).map(p => `${p.name}@${p.version}`)) + } + + if (opt?.isModule) { + if (!opt.publishName) { + throw new Error(`Expected module to have a publish name`) + } + + const data3 = extractPointers(normalizeSymbolIds(data)) + // const data2 = serializePointers(data) + const datafile = { + kind: 'deployed' as const, + table: data3[0].table, + captured: data3[0].captured, + } + + const p = await buildFs.writeData2(datafile, { source: target, dependencies, packageDependencies, pointers: data3[1] }) + const text = `module.exports = require('${p}');` + + return { + extname, + location: await buildFs.writeFile(opt.publishName, text, { dependencies: [p] }), + } + } + + // TODO: implement hash tree for integrity checks against the 'data' files + + async function saveArtifact(data: Uint8Array, name: string, source: string, pointers?: any) { + const p = await buildFs.writeData(data, { name, source, dependencies, packageDependencies, pointers }) + if (opt?.publishName) { + if (!isArtifact) { + return buildFs.writeFile(opt.publishName, data) + } + + const text = `module.exports = require('${p}');` + await buildFs.writeFile(opt.publishName, text, { dependencies: [p] }) + + return p + } + + return p + } + + if (isArtifact) { + const name = path.relative(workingDirectory, outfile) + const data3 = extractPointers(normalizeSymbolIds(data)) + + const datafile = { + kind: 'deployed' as const, + table: data3[0].table, + captured: data3[0].captured, + } + + if (!opt?.publishName) { + return { + extname, + location: await buildFs.writeData2(datafile, { source: target, dependencies, packageDependencies, pointers: data3[1] }) + } + } + + const artifactData = Buffer.from(JSON.stringify(datafile), 'utf-8') + + return { + extname, + location: await saveArtifact(artifactData, name, target, data3[1]), + } + } + + // TODO: we should still add `packageDependencies` even when bundled if modules were marked external + const importMap = bundled ? await getImportMap(ctx, data.table) : undefined + const moduleResolver = ctx.createModuleResolver() + if (importMap) { + getLogger().debug('Registering import map for bundling', Object.keys(importMap)) + moduleResolver.registerMapping(importMap) + } + + compilerOptions.alwaysStrict ??= true + + // FIXME: emit source map separately and attach it as metadata to the artifact instead of inlining + const transpiler = createTranspiler( + toFs(workingDirectory, buildFs, ctx.fs), + moduleResolver, + compilerOptions, + ) + + const optimizer = opt?.useOptimizer ? createOptimizer(buildFs) : undefined + const serializerHost = opt?.includeAssets ? createSerializerHost(buildFs, 'backend-bundle', optimizer) : createPointerMapper() + const sourceFile = renderFile(data, opt?.platform, bundled, opt?.immediatelyInvoke, undefined, isArtifact, serializerHost) + const sourceFileName = outfile.replace(/\.(?:m)?j(sx?)$/, '.t$1') + + const res = await transpiler.transpile( + sourceFileName, + sourceFile, + outfile, + { + workingDirectory, + bundleOptions: { ...opt, bundled, serializerHost, minifyKeepWhitespace: true }, + } + ) + + const pointer = await saveArtifact( + res.result.contents, + path.relative(workingDirectory, outfile), + target, + ) + + const assets = opt?.includeAssets ? (serializerHost as ReturnType).getAssets() : undefined + + if (opt?.destination === undefined) { + return { + assets, + extname, + location: pointer, + } + } + + return { + assets, + extname, + location: outfile, + } +} + +export function isDeduped(obj: any): obj is { captured: any; table: Record } { + return typeof obj === 'object' && !!obj && obj['__isDeduped'] +} + +export function isNormalized(obj: any): obj is ReturnType & { __isNormalized: true } { + return typeof obj === 'object' && !!obj && obj['__isNormalized'] +} + +export function createDataExport(captured: any) { + if (isDeduped(captured)) { + return captured + } + + const deduped = createDataTable(captured) + + return { + table: deduped.table, + captured: deduped.captured, + } +} diff --git a/src/codegen/openapi.ts b/src/codegen/openapi.ts new file mode 100644 index 0000000..be6774f --- /dev/null +++ b/src/codegen/openapi.ts @@ -0,0 +1,13 @@ +import ts from 'typescript' + +// Only doing v3 + +interface XGitHub { + githubCloudOnly: boolean + enabledForGitHubApps: boolean + category: string + subcategory: string +} + + +// https://unpkg.com/browse/@octokit/openapi/generated/ diff --git a/src/codegen/openapiv3.ts b/src/codegen/openapiv3.ts new file mode 100644 index 0000000..0ef4dc4 --- /dev/null +++ b/src/codegen/openapiv3.ts @@ -0,0 +1,267 @@ +interface Info { + readonly title: string; + readonly description?: string; + readonly termsOfService?: string; + readonly contact?: Contact; + readonly license?: License; + readonly version: string; + readonly [name: `x-${string}`]: any | undefined; +} +interface Contact { + readonly name?: string; + readonly url?: string; + readonly email?: string; + readonly [name: `x-${string}`]: any | undefined; +} +interface License { + readonly name: string; + readonly url?: string; + readonly [name: `x-${string}`]: any | undefined; +} +interface ExternalDocumentation { + readonly description?: string; + readonly url: string; + readonly [name: `x-${string}`]: any | undefined; +} +interface Server { + readonly url: string; + readonly description?: string; + readonly variables?: Record; + readonly [name: `x-${string}`]: any | undefined; +} +interface ServerVariable { + readonly enum?: string[]; + readonly default: string; + readonly description?: string; + readonly [name: `x-${string}`]: any | undefined; +} +interface Tag { + readonly name: string; + readonly description?: string; + readonly externalDocs?: ExternalDocumentation; + readonly [name: `x-${string}`]: any | undefined; +} +interface Paths { + readonly [name: `/${string}`]: PathItem | undefined; + readonly [name: `x-${string}`]: any | undefined; +} +interface PathItem { + readonly $ref?: string; + readonly summary?: string; + readonly description?: string; + readonly servers?: Server[]; + readonly parameters?: ((any & SchemaXORContent & ParameterLocation) | Reference)[]; + readonly "get"?: Operation; + readonly "put"?: Operation; + readonly "post"?: Operation; + readonly "delete"?: Operation; + readonly "options"?: Operation; + readonly "head"?: Operation; + readonly "patch"?: Operation; + readonly "trace"?: Operation; + readonly [name: `x-${string}`]: any | undefined; +} +type SchemaXORContent = (any & any & any & any & any); +type ParameterLocation = { + readonly in?: "path"; + readonly style?: "matrix" | "label" | "simple"; + readonly required: true; +} | { + readonly in?: "query"; + readonly style?: "form" | "spaceDelimited" | "pipeDelimited" | "deepObject"; +} | { + readonly in?: "header"; + readonly style?: "simple"; +} | { + readonly in?: "cookie"; + readonly style?: "form"; +}; +interface Reference { + readonly "$ref"?: string; +} +interface Operation { + readonly tags?: string[]; + readonly summary?: string; + readonly description?: string; + readonly externalDocs?: ExternalDocumentation; + readonly operationId?: string; + readonly parameters?: ((any & SchemaXORContent & ParameterLocation) | Reference)[]; + readonly requestBody?: RequestBody | Reference; + readonly responses: Responses; + readonly callbacks?: Record | Reference>; + readonly deprecated?: boolean; + readonly security?: Record[]; + readonly servers?: Server[]; + readonly [name: `x-${string}`]: any | undefined; +} +interface RequestBody { + readonly description?: string; + readonly content: Record; + readonly required?: boolean; + readonly [name: `x-${string}`]: any | undefined; +} +interface Responses { + readonly default?: Response | Reference; + readonly [name: `${string}`]: (Response | Reference) | undefined; + readonly [name: `x-${string}`]: any | undefined; +} +interface Response { + readonly description: string; + readonly headers?: Record; + readonly content?: Record; + readonly links?: Record; + readonly [name: `x-${string}`]: any | undefined; +} +interface Components { + readonly schemas?: { + readonly [name: `${string}`]: (Schema | Reference) | undefined; + }; + readonly responses?: { + readonly [name: `${string}`]: (Reference | Response) | undefined; + }; + readonly parameters?: { + readonly [name: `${string}`]: (Reference | (any & SchemaXORContent & ParameterLocation)) | undefined; + }; + readonly examples?: { + readonly [name: `${string}`]: (Reference | Example) | undefined; + }; + readonly requestBodies?: { + readonly [name: `${string}`]: (Reference | RequestBody) | undefined; + }; + readonly headers?: { + readonly [name: `${string}`]: (Reference | (any & SchemaXORContent)) | undefined; + }; + readonly securitySchemes?: { + readonly [name: `${string}`]: (Reference | SecurityScheme) | undefined; + }; + readonly links?: { + readonly [name: `${string}`]: (Reference | any) | undefined; + }; + readonly callbacks?: { + readonly [name: `${string}`]: (Reference | Record) | undefined; + }; + readonly [name: `x-${string}`]: any | undefined; +} +interface Schema { + readonly title?: string; + readonly multipleOf?: number; + readonly maximum?: number; + readonly exclusiveMaximum?: boolean; + readonly minimum?: number; + readonly exclusiveMinimum?: boolean; + readonly maxLength?: number; + readonly minLength?: number; + readonly pattern?: string; + readonly maxItems?: number; + readonly minItems?: number; + readonly uniqueItems?: boolean; + readonly maxProperties?: number; + readonly minProperties?: number; + readonly required?: string[]; + readonly enum?: any[]; + readonly type?: "array" | "boolean" | "integer" | "number" | "object" | "string"; + readonly not?: Schema | Reference; + readonly allOf?: (Schema | Reference)[]; + readonly oneOf?: (Schema | Reference)[]; + readonly anyOf?: (Schema | Reference)[]; + readonly items?: Schema | Reference; + readonly properties?: Record; + readonly additionalProperties?: Schema | Reference | boolean; + readonly description?: string; + readonly format?: string; + readonly default?: any; + readonly nullable?: boolean; + readonly discriminator?: Discriminator; + readonly readOnly?: boolean; + readonly writeOnly?: boolean; + readonly example?: any; + readonly externalDocs?: ExternalDocumentation; + readonly deprecated?: boolean; + readonly xml?: XML; + readonly [name: `x-${string}`]: any | undefined; +} +interface Discriminator { + readonly propertyName: string; + readonly mapping?: Record; +} +interface XML { + readonly name?: string; + readonly namespace?: string; + readonly prefix?: string; + readonly attribute?: boolean; + readonly wrapped?: boolean; + readonly [name: `x-${string}`]: any | undefined; +} +interface Example { + readonly summary?: string; + readonly description?: string; + readonly value?: any; + readonly externalValue?: string; + readonly [name: `x-${string}`]: any | undefined; +} +type SecurityScheme = APIKeySecurityScheme | HTTPSecurityScheme | OAuth2SecurityScheme | OpenIdConnectSecurityScheme; +interface APIKeySecurityScheme { + readonly type: "apiKey"; + readonly name: string; + readonly in: "header" | "query" | "cookie"; + readonly description?: string; + readonly [name: `x-${string}`]: any | undefined; +} +type HTTPSecurityScheme = { + readonly scheme?: string; +} | any; +interface OAuth2SecurityScheme { + readonly type: "oauth2"; + readonly flows: OAuthFlows; + readonly description?: string; + readonly [name: `x-${string}`]: any | undefined; +} +interface OAuthFlows { + readonly implicit?: ImplicitOAuthFlow; + readonly password?: PasswordOAuthFlow; + readonly clientCredentials?: ClientCredentialsFlow; + readonly authorizationCode?: AuthorizationCodeOAuthFlow; + readonly [name: `x-${string}`]: any | undefined; +} +interface ImplicitOAuthFlow { + readonly authorizationUrl: string; + readonly refreshUrl?: string; + readonly scopes: Record; + readonly [name: `x-${string}`]: any | undefined; +} +interface PasswordOAuthFlow { + readonly tokenUrl: string; + readonly refreshUrl?: string; + readonly scopes: Record; + readonly [name: `x-${string}`]: any | undefined; +} +interface ClientCredentialsFlow { + readonly tokenUrl: string; + readonly refreshUrl?: string; + readonly scopes: Record; + readonly [name: `x-${string}`]: any | undefined; +} +interface AuthorizationCodeOAuthFlow { + readonly authorizationUrl: string; + readonly tokenUrl: string; + readonly refreshUrl?: string; + readonly scopes: Record; + readonly [name: `x-${string}`]: any | undefined; +} +interface OpenIdConnectSecurityScheme { + readonly type: "openIdConnect"; + readonly openIdConnectUrl: string; + readonly description?: string; + readonly [name: `x-${string}`]: any | undefined; +} +export interface OpenApiModel { + readonly openapi: string; + readonly info: Info; + readonly externalDocs?: ExternalDocumentation; + readonly servers?: Server[]; + readonly security?: Record[]; + readonly tags?: Tag[]; + readonly paths: Paths; + readonly components?: Components; + readonly [name: `x-${string}`]: any | undefined; +} \ No newline at end of file diff --git a/src/codegen/providers.ts b/src/codegen/providers.ts new file mode 100644 index 0000000..6f25846 --- /dev/null +++ b/src/codegen/providers.ts @@ -0,0 +1,789 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { getProviderCacheDir, getProviderTypesDirectory, getWorkingDirectory, getGlobalCacheDirectory } from '../workspaces' +import { capitalize, memoize, toAmbientDeclarationFile, toSnakeCase } from '../utils' +import { providerPrefix } from '../runtime/loader' +import { Fs, SyncFs } from '../system' +import { runCommand } from '../utils/process' + +// --------------------------------------------------------------- // +// --------------------- SCHEMA RENDERING ------------------------ // +// --------------------------------------------------------------- // + +interface TerraformSchema { + readonly format_version: '1.0' | string + readonly provider_schemas: Record +} + +interface ProviderSchema { + readonly provider: Schema + readonly resource_schemas: Record + readonly data_source_schemas: Record +} + +interface Schema { + readonly version: 0 | number // schema version, not provider + readonly block: Block +} + +type DescriptionKind = 'plain' | 'markdown' | string + +interface Block { + readonly attributes?: Record + readonly block_types?: Record + readonly description?: string + readonly description_kind?: DescriptionKind +} + +type AttributeType = PrimitiveType | ObjectType | ContainerType +type PrimitiveType = 'string' | 'bool' | 'number' | 'dynamic' +type ObjectType = ['object', Record] +type ContainerType = ['set' | 'map' | 'list', AttributeType] + +// It appears that if the type is a `list` then the the next element in the array is the element type +// Same thing applies to `map` and `set` and `object` ? + +interface Attribute { + readonly type: AttributeType + readonly description?: string + readonly description_kind?: DescriptionKind + readonly required?: boolean + readonly computed?: boolean + readonly optional?: boolean + readonly deprecated?: boolean + readonly sensitive?: boolean +} + +interface BlockType { + readonly nesting_mode: 'single' | 'set' | 'list' | 'map' + readonly block: Block + readonly min_items?: number + readonly max_items?: number // only in `list` ? +} + +function getImmediateObjectType(v: AttributeType): ObjectType[1] | undefined { + if (!Array.isArray(v)) { + return + } + + if (v[0] === 'object') { + return v[1] + } + + // if (Array.isArray(v[1]) && v[1][0] === 'object') { + // return v[1][1] + // } + + return +} + +function hasRequiredField(block: Block) { + if (!block.attributes) return false + + return !!Object.values(block.attributes).find(a => a.required || !a.optional) +} + +function createNameMapper(mappings = Object.create(null) as Record) { + function normalize(str: string) { + const [first, ...rest] = str.split('_') + const mapped = [first, ...rest.map(capitalize)].join('') + if (typeof mappings[mapped] === 'object') { + mappings[mapped][''] = toSnakeCase(mapped) !== str ? str : '' + } else { + mappings[mapped] = toSnakeCase(mapped) !== str ? str : '' + } + + return [first, ...rest.map(capitalize)].join('') + } + + function addType(str: string, type: string) { + const [first, ...rest] = str.split('_') + const mapped = [first, ...rest.map(capitalize)].join('') + if (typeof mappings[mapped] === 'string') { + mappings[mapped] = { '': mappings[mapped] } + } else if (!mappings[mapped]) { + mappings[mapped] = { '': '' } + } + + mappings[mapped]['__type'] = type + } + + function createSubmapper(suffix: string) { + const submappings = mappings[suffix] = typeof mappings[suffix] === 'string' + ? { '': mappings[suffix] } + : (typeof mappings[suffix] === 'undefined' ? {} : mappings[suffix]) + + return createNameMapper(submappings) + } + + return { mappings, normalize, createSubmapper, addType } +} + +function inheritDocs(to: T, from: ts.Node) { + const comments = ts.getSyntheticLeadingComments(from) + ts.setSyntheticLeadingComments(to, comments) + + return to +} + +const savedDocs = new Map() +function addDocs(node: T, docs?: string, key?: string) { + // Some data sources are directly related with a resource but do not include docs in their schema + if (key) { + if (docs) { + savedDocs.set(key, docs) + } else { + docs = inferDocs(key) + } + } + + if (!docs) return node + + const lines = docs.split('\n') + const text = `*\n${lines.map(l => ` * ${l.trim()}`).join('\n')}\n ` + + return ts.setSyntheticLeadingComments(node, [{ + text, + pos: -1, + end: -1, + hasLeadingNewline: true, + hasTrailingNewLine: true, + kind: ts.SyntaxKind.MultiLineCommentTrivia, + }]) +} + +function inferDocs(key: string) { + const [name, prop] = key.split('.') + + if (name.endsWith('Data')) { + return savedDocs.get(`${name.slice(0, -4)}.${prop}`) + } +} + +function createGenerator( + prefix = '', + mapper = createNameMapper(), + interfaces = new Map() +) { + function createBlockType(name: string, data: BlockType, variance: 'in' | 'out' = 'in'): ts.PropertySignature { + const key = `${prefix}.${name}-${variance}` + if (!interfaces.has(key)) { + const declName = `${prefix}${capitalize(name)}` + const generator = createGenerator(declName, mapper.createSubmapper(name), interfaces) + const elements = generator.getBlockElements(data.block, variance === 'in' ? { excludeComputed: true } : undefined) + const decl = ts.factory.createInterfaceDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], + variance === 'out' ? `${declName}OutProps` : `${declName}Props`, + undefined, + undefined, + elements + ) + + interfaces.set(key, decl) + } + + const ref = ts.factory.createTypeReferenceNode(interfaces.get(key)!.name) + + // TODO: if `max_items` is 1 then we shouldn't create an array node + + return ts.factory.createPropertySignature( + [ts.factory.createModifier(ts.SyntaxKind.ReadonlyKeyword)], + name, + // I guess it's always optional? + !data.min_items ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) : undefined, + // hasRequiredField(data.block) ? undefined : ts.factory.createToken(ts.SyntaxKind.QuestionToken), + (data.nesting_mode === 'list' || data.nesting_mode === 'set') + ? ts.factory.createArrayTypeNode(ref) + : data.nesting_mode === 'map' + ? ts.factory.createTypeReferenceNode('Record', [ts.factory.createTypeReferenceNode('string'), ref]) + : ref + ) + } + + function createAttribute(name: string, data: Attribute, variance: 'in' | 'out' = 'in') { + const isOptional = variance === 'in' + ? !(data.required || (!data.optional && !data.computed)) + : (!data.required && !data.computed) + + const prop = ts.factory.createPropertySignature( + [ts.factory.createModifier(ts.SyntaxKind.ReadonlyKeyword)], + name, + isOptional ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) : undefined, + createAttributeShape(data.type, name) + ) + + return addDocs(prop, data.description, `${prefix}.${name}`) + } + + function getObjectMembers(o: any) { + const members: ts.TypeElement[] = [] + for (const [k, v] of Object.entries(o)) { + members.push(ts.factory.createPropertySignature( + [ts.factory.createModifier(ts.SyntaxKind.ReadonlyKeyword)], + mapper.normalize(k), + undefined, + createAttributeShape(v as any, k) + )) + } + + return members + } + + // Name is the name of the associated member + function createAttributeShape(type: AttributeType, name: string): ts.TypeNode { + const objectType = getImmediateObjectType(type) + if (objectType) { + const key = `${prefix}.${name}` + if (!interfaces.has(key)) { + const declName = `${prefix}${capitalize(name)}` + const generator = createGenerator(declName, mapper.createSubmapper(name), interfaces) + const elements = generator.getObjectMembers(objectType) + const decl = ts.factory.createInterfaceDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], + `${declName}Props`, + undefined, + undefined, + elements + ) + + interfaces.set(key, decl) + } + + return ts.factory.createTypeReferenceNode(interfaces.get(key)!.name) + } + + if (typeof type === 'string') { + switch (type) { + case 'string': + case 'number': + return ts.factory.createTypeReferenceNode(type) + case 'bool': + return ts.factory.createTypeReferenceNode('boolean') + case 'dynamic': + return ts.factory.createTypeReferenceNode('any') + default: + throw new Error(`Unknown type: ${type}`) + } + } + + switch (type[0]) { + case 'object': + return ts.factory.createTypeLiteralNode(getObjectMembers(type[1])) + case 'set': + case 'list': + return ts.factory.createArrayTypeNode(createAttributeShape(type[1], name)) + case 'map': + return ts.factory.createTypeReferenceNode('Record', [ + ts.factory.createTypeReferenceNode('string'), + createAttributeShape(type[1], name) + ]) + } + } + + function getBlockElements(data: Block, opt?: { excludeComputed?: boolean }): ts.PropertySignature[] { + const members: ts.PropertySignature[] = [] + const variance = opt?.excludeComputed ? 'in' : 'out' + if (data.attributes) { + for (const [k, v] of Object.entries(data.attributes)) { + if (v.computed && !v.required && !v.optional && opt?.excludeComputed) continue + + const name = mapper.normalize(k) + members.push(createAttribute(name, v, variance)) + } + } + + if (data.block_types) { + for (const [k, v] of Object.entries(data.block_types)) { + const name = mapper.normalize(k) + members.push( + addDocs( + createBlockType(name, v, variance), + v.block.description, + `${prefix}.${name}` + ) + ) + } + } + + return members + } + + return { mapper, interfaces, getBlockElements, getObjectMembers } +} + +function generateConstruct(name: string, schema: Schema, mapper = createNameMapper(), kind?: 'provider' | 'resource' | 'data-source') { + const propsName = `${name}Props` + const generator = createGenerator(name, mapper) + // Must be called first + removeNestedTypes(schema.block) + + const stateElements = generator.getBlockElements(schema.block) + const propsDeclElements = generator.getBlockElements(schema.block, { excludeComputed: true }) + const result: (ts.InterfaceDeclaration | ts.ClassDeclaration)[] = [...generator.interfaces.values()] + + function createConstructorParameters() { + if (propsDeclElements.length === 0) { + return [] + } + + const propsDecl = ts.factory.createInterfaceDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], + propsName, + undefined, + undefined, + propsDeclElements + ) + result.push(propsDecl) + + const props = ts.factory.createParameterDeclaration( + undefined, + undefined, + 'props', + propsDeclElements.filter(x => !x.questionToken).length === 0 + ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) + : undefined, + ts.factory.createTypeReferenceNode(propsName) + ) + + return [props] + } + + const ctor = ts.factory.createConstructorDeclaration( + undefined, + createConstructorParameters(), + undefined + ) + + function createPropertyDecl(sig: ts.PropertySignature) { + const decl = ts.factory.createPropertyDeclaration( + (sig as any).modifiers, + (sig.name as ts.Identifier), + sig.questionToken, + sig.type, + undefined + ) + + return inheritDocs(decl, sig) + } + + // Only static provider attributes can be referenced + const props = (kind === 'provider' ? propsDeclElements : stateElements).map(createPropertyDecl) + + const decl = ts.factory.createClassDeclaration( + [ + ts.factory.createModifier(ts.SyntaxKind.ExportKeyword), + ts.factory.createModifier(ts.SyntaxKind.DeclareKeyword) + ], + name, + undefined, + undefined, + [...props, ctor] + ) + + result.push(addDocs(decl, schema.block.description)) + + return result +} + +// Exclude `Wafv2` ??? +// It's massive... + +async function installTypes(fs: Fs, workingDirectory: string, packages: Record) { + const indexFileName = 'index.d.ts' + const typesPackage = getProviderTypesDirectory(workingDirectory) + const packageJsonPath = path.resolve(typesPackage, 'package.json') + + async function getPackageJson(): Promise<{ name: string; types: string; installed: Record }> { + try { + return JSON.parse(await fs.readFile(packageJsonPath, 'utf-8')) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + + return { + name: '@types/synapse-providers', + types: indexFileName, + installed: {}, + } + } + } + + const getDeclFileName = (providerName: string) => `${providerName}.d.ts` + + const packageJson = await getPackageJson() + packageJson.installed ??= {} + + await Promise.all(Object.entries(packages).map(async ([k, v]) => { + const typesFile = await fs.readFile(path.resolve(v, indexFileName)) + await fs.writeFile(path.resolve(typesPackage, `${k}.d.ts`), typesFile) + packageJson.installed[k] = true + })) + + packageJson.installed = Object.fromEntries( + Object.entries(packageJson.installed).sort((a, b) => a[0].localeCompare(b[0])) + ) + + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, undefined, 4)) + + const indexFile = Object.entries(packageJson.installed) + .filter(([_, v]) => v) + .map(([k]) => getDeclFileName(k)) + .map(n => `/// `).join('\n') + + await fs.writeFile( + path.resolve(typesPackage, indexFileName), + indexFile, + ) +} + +async function generateProvider(fs: Fs, provider: ProviderInfo, outdir: string, generateLazyBindings = true) { + interface Provider { + readonly kind: 'provider' + readonly id: string + readonly source: string + readonly version: string + } + + interface Resource { + readonly kind: 'resource' + readonly id: string + } + + interface DataSource { + readonly kind: 'data-source' + readonly id: string + } + + type Element = Provider | Resource | DataSource + + const mapper = createNameMapper() + const map = new Map() + const outfile: ts.Statement[] = [] + + + const providerName = `${capitalize(provider.name)}Provider` + outfile.push(...generateConstruct(providerName, provider.schema.provider, mapper.createSubmapper(providerName), 'provider')) + map.set(providerName, { + kind: 'provider', + id: provider.name, + source: provider.source, + version: provider.version, + }) + + + for (const [k, v] of Object.entries(provider.schema.resource_schemas)) { + // we'll remove the provider name + const name = capitalize(mapper.normalize(k.split('_').slice(1).join('_'))) + outfile.push(...generateConstruct(name, v, mapper.createSubmapper(name))) + map.set(name, { id: k, kind: 'resource' }) + } + + for (const [k, v] of Object.entries(provider.schema.data_source_schemas)) { + const name = capitalize(mapper.normalize(k.split('_').slice(1).join('_'))) + 'Data' + outfile.push(...generateConstruct(name, v, mapper.createSubmapper(name))) + map.set(name, { id: k, kind: 'data-source' }) + } + + const declSf = ts.factory.createSourceFile(outfile, ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), ts.NodeFlags.None) + const sf = toAmbientDeclarationFile(`${providerPrefix}${provider.name}`, declSf) + + // Actual logic + const indexfile: string[] = [] + indexfile.push('"use strict";') + indexfile.push('const terraform = require("synapse:terraform");') + + if (generateLazyBindings) { + indexfile.push('function bindClass(name, fn) {') + indexfile.push(' let v;') + indexfile.push(' const get = () => v ??= terraform.createTerraformClass(...fn());') + indexfile.push(' Object.defineProperty(exports, name, { get });') + indexfile.push('}') + } else { + indexfile.push('function bindClass(name, type, kind, mappings, version, source) {') + indexfile.push(' exports[name] = terraform.createTerraformClass(type, kind, mappings, version, source);') + indexfile.push('}') + } + + for (const [k, v] of map.entries()) { + const mappings = mapper.mappings[k] + const args: any[] = [k, v.id, v.kind] + if (typeof mappings !== 'object' || Object.keys(mappings).length === 0) { + args.push(undefined); + } else { + args.push(mappings) + } + + if (v.kind === 'provider') { + args.push(v.version, v.source) + } + + if (generateLazyBindings) { + indexfile.push(`bindClass('${args[0]}', () => JSON.parse('${JSON.stringify(args.slice(1))}'));`) + } else { + indexfile.push(`bindClass(...JSON.parse('${JSON.stringify(args)}'));`) + } + } + + await fs.writeFile( + path.resolve(outdir, 'index.js'), + indexfile.join('\n') + ) + + await fs.writeFile(path.resolve(outdir, 'index.d.ts'), sf.text) + + await createResolvedFile() + + const packageJson = { + name: provider.name, + source: provider.source, + version: provider.version, + exports: { '.': './index.js' }, + } + + await fs.writeFile( + path.resolve(outdir, 'package.json'), + JSON.stringify(packageJson, undefined, 4) + ) + + return outdir + + // Used for bundling + // Can be removed once these classes are stripped out from the serialization data on deployment + async function createResolvedFile() { + const resolvedFile: string[] = [] + resolvedFile.push('"use strict";') + resolvedFile.push(`const names = JSON.parse('${JSON.stringify(Array.from(map.keys()))}');`) + resolvedFile.push(`names.forEach(name => exports[name] = class {});`) + + await fs.writeFile( + path.resolve(outdir, 'index.resolved.js'), + resolvedFile.join('\n') + ) + } +} + +function render(o: any): string { + if (typeof o === 'string') { + return `'${o}'` + } + if (typeof o === 'undefined') { + return 'undefined' + } + if (typeof o !== 'object' || !o) { + return o + } + + return `{ ${Object.entries(o).map(([k, v]) => `'${k}': ${render(v)}`).join(', ')} }` +} + +async function getProviderVersions(terraformPath: string, cwd: string) { + const result: Record = {} + const stdout = await runCommand(terraformPath, ['version'], { cwd }) + + for (const line of stdout.split('\n')) { + const [_, source, version] = line.match(/provider (.*) v([0-9]+\.[0-9]+\.[0-9]+.*)/) ?? [] + if (source && version) { + result[source] = version + } + } + + return result +} + +function removeNestedTypes(block: Block) { + if (!block.attributes) { + return block + } + + const block_types = (block as { -readonly [P in keyof Block]: Block[P] }).block_types ??= {} + for (const [k, v] of Object.entries(block.attributes)) { + if ('nested_type' in v) { + delete block.attributes[k] + block_types[k] = { + block: { + ...removeNestedTypes((v as any).nested_type), + description: v.description, + description_kind: v.description_kind, + optional: v.optional, + } as any, + nesting_mode: (v as any).nested_type.nesting_mode, + } + } + } + + return block +} + +async function listVersions( + terraformPath: string, + source: string, +): Promise { + const result = await runCommand(terraformPath, ['providers', 'list', source]) + const data = JSON.parse(result.trim().split('\n').pop()!).data + if (!Array.isArray(data)) { + throw new Error(`Expected an array of versions, got: ${typeof data}`) + } + + // Latest version first + return data.reverse() +} + +async function getProviderSchema( + fs: Fs & SyncFs, + terraformPath: string, + info: Omit, + cwd = process.cwd(), +): Promise { + const { name, source, version } = info + const tfJson = JSON.stringify({ + terraform: { + 'required_providers': { + [name]: { version, source } + } + } + }) + + const tfJsonPath = path.resolve(cwd, 'tmp.tf.json') + await fs.writeFile(tfJsonPath, tfJson) + const env = { TF_PLUGIN_CACHE_DIR: getProviderCacheDir() } + + try { + await runCommand(terraformPath, ['init'], { cwd, env }) + const stdout = await runCommand(terraformPath, ['providers', 'schema', '-json'], { cwd, env }) + + const parsed = JSON.parse(stdout) as TerraformSchema + const schema = parsed['provider_schemas'][source] + if (!schema) { + throw new Error(`Failed to get provider schema for "${name}" from Terraform output`) + } + + return { + name, + source, + schema, + version, + } + } finally { + try { + await fs.deleteFile(tfJsonPath) + await fs.deleteFile(path.resolve(cwd, '.terraform.lock.hcl')) + } catch {} + } +} + +export interface ProviderConfig { + readonly name: string + readonly source?: string + readonly version?: string +} + +interface ResolvedProviderConfig extends ProviderConfig { + readonly source: string + readonly version: string +} + +interface ProviderInfo extends ResolvedProviderConfig { + readonly schema: ProviderSchema +} + +export function getProviderSource(name: string, providerRegistryHostname?: string) { + switch (name) { + // case 'aws': + // return `${providerRegistryHostname}/cohesible/aws` + case 'github': + return 'registry.terraform.io/integrations/github' + case 'fly': + return 'registry.terraform.io/fly-apps/fly' + case 'kubernetes': + return 'registry.terraform.io/hashicorp/kubernetes' + + // Built-ins + case 'synapse': + case 'terraform': + return `terraform.io/builtin/${name}` + + default: + return `registry.terraform.io/hashicorp/${name}` + } +} + +export async function listProviderVersions(name: string, providerRegistryHostname?: string, terraformPath = 'terraform') { + const source = name.includes('/') ? name : getProviderSource(name, providerRegistryHostname) + + return listVersions(terraformPath, source) +} + +export function createProviderGenerator(fs: Fs & SyncFs, providerRegistryHostname: string, terraformPath = 'terraform') { + const providersDir = path.resolve(getGlobalCacheDirectory(), 'providers-codegen') + const schemasDir = path.resolve(providersDir, 'schemas') + const packagesDir = path.resolve(providersDir, 'packages') + + async function resolveProvider(provider: ProviderConfig): Promise { + const source = provider.source ?? getProviderSource(provider.name, providerRegistryHostname) + const version = provider.version ?? (await listVersions(terraformPath, source))[0] + + return { + source, + version, + name: provider.name, + } + } + + async function getSchema(resolved: ResolvedProviderConfig): Promise { + const key = `${resolved.source}/${resolved.version}` + const cacheDir = path.resolve(schemasDir, '.cache') + const schemaPath = path.resolve(cacheDir, key, 'schema.json') + + try { + return JSON.parse(await fs.readFile(schemaPath, 'utf-8')) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + + const schemasWorkingDir = path.resolve(schemasDir, `${resolved.name}-${resolved.version}`) + const info = await getProviderSchema(fs, terraformPath, resolved, schemasWorkingDir) + await fs.writeFile(schemaPath, JSON.stringify(info.schema)) + + return info.schema + } + } + + const getInstalledProviders = memoize(async function (): Promise> { + const pkgManifest = path.resolve(providersDir, 'manifest.json') + + try { + return JSON.parse(await fs.readFile(pkgManifest, 'utf-8')) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + return {} + } + }) + + async function generate(provider: ProviderConfig, dest?: string) { + const resolved = await resolveProvider(provider) + const info = { ...resolved, schema: await getSchema(resolved) } + const outdir = dest ?? path.resolve(packagesDir, resolved.source, resolved.version) + + const pkgManifest = path.resolve(providersDir, 'manifest.json') + const manifest = await getInstalledProviders() + + const pkgDest = await generateProvider(fs, info, outdir) + manifest[provider.name] = path.relative(providersDir, pkgDest) + await fs.writeFile(pkgManifest, JSON.stringify(manifest, undefined, 4)) + + return { + ...resolved, + dest: pkgDest, + } + } + + return { + generate, + resolveProvider, + installTypes: (packages: Record, workingDirectory = getWorkingDirectory()) => installTypes(fs, workingDirectory, packages), + } +} diff --git a/src/codegen/schemas.ts b/src/codegen/schemas.ts new file mode 100644 index 0000000..856903a --- /dev/null +++ b/src/codegen/schemas.ts @@ -0,0 +1,614 @@ +import ts from 'typescript' +import { isNonNullable, printNodes } from '../utils' + +// Rough impl. of JSONSchema7 + +type PrimitiveType = 'null' | 'boolean' | 'object' | 'array' | 'number' | 'string' | 'integer' +type Literal = null | boolean | string | number + +interface SchemaBase { + readonly type?: PrimitiveType | PrimitiveType[] + readonly enum?: Literal[] + readonly const?: Literal + readonly oneOf?: Schema[] + readonly title?: string +} + +interface ObjectSchema extends SchemaBase { + readonly type: 'object' + readonly properties?: Record + readonly required?: string[] + readonly additionalProperties?: Schema + readonly maxProperties?: number + readonly minProperties?: number + readonly dependentRequired?: Record + readonly patternProperties?: Record +} + +interface ArraySchema extends SchemaBase { + readonly type: 'array' + readonly items?: Schema | false + readonly prefixItems?: Schema[] + readonly maxItems?: number + readonly minItems?: number + readonly uniqueItems?: boolean +} + +interface StringSchema extends SchemaBase { + readonly type: 'string' + readonly pattern?: string // RegExp + readonly maxLength?: number + readonly minLength?: number +} + +interface NumberSchema extends SchemaBase { + readonly type: 'number' | 'integer' + readonly multipleOf?: number + readonly maximum?: number + readonly minimum?: number + readonly exclusiveMaximum?: number + readonly exclusiveMinimum?: number +} + +interface BooleanSchema extends SchemaBase { + readonly type: 'boolean' +} + +interface NullSchema extends SchemaBase { + readonly type: 'null' +} + +interface Ref { + readonly $ref: string +} + +interface OneOf extends SchemaBase { + readonly oneOf: Schema[] +} + +interface AllOf extends SchemaBase { + readonly allOf: Schema[] +} + +interface AnyOf extends SchemaBase { + readonly anyOf: Schema[] +} + +export type Schema = ObjectSchema | ArraySchema | StringSchema | NumberSchema | BooleanSchema | NullSchema | OneOf | AllOf | Ref | boolean + +function isRef(o: any): o is Ref { + return typeof o === 'object' && !!o && typeof o.$ref === 'string' +} + +function isOneOf(o: any): o is OneOf { + return typeof o === 'object' && !!o && Array.isArray(o.oneOf) +} + +function isAllOf(o: any): o is AllOf { + return typeof o === 'object' && !!o && Array.isArray(o.allOf) +} + +function isAnyOf(o: any): o is AnyOf { + return typeof o === 'object' && !!o && Array.isArray(o.anyOf) +} + +function isObjectSchema(o: any): o is ObjectSchema { + return typeof o === 'object' && !!o && o.type === 'object' +} + +function quote(name: string) { + if (name.match(/^[\-\+_]/)) { + return `'${name}'` + } + + return name +} + +function tryConvertRegExpPattern(pattern: string) { + const strings: string[] = [] + const types: ts.TypeNode[] = [] + + let isLiteral = true + + if (!pattern.startsWith('^')) { + strings.push('') + types.push(ts.factory.createKeywordTypeNode(ts.SyntaxKind.StringKeyword)) + isLiteral = false + } else { + pattern = pattern.slice(1) + } + + pattern = pattern.replace(/\\([\$\/])/g, '$1') + + while (pattern) { + const g = pattern.indexOf('(') + const g2 = pattern.indexOf('[') + if (g2 !== -1) { + let end = pattern.indexOf(']') + if (end === -1) { + throw new Error(`Bad parse: missing end of group token: ${pattern}`) + } + + if (pattern[end + 1] === '+') { + end++ + } + + strings.push(pattern.slice(0, g2)) + types.push(ts.factory.createKeywordTypeNode(ts.SyntaxKind.StringKeyword)) + isLiteral = false + + pattern = pattern.slice(end + 1) + continue + } + if (g === -1) { + if (!pattern.endsWith('$')) { + types.push(ts.factory.createKeywordTypeNode(ts.SyntaxKind.StringKeyword)) + isLiteral = false + } else { + pattern = pattern.slice(0, -1) + } + + strings.push(pattern) + pattern = '' + continue + } + const end = pattern.indexOf(')') + if (end === -1) { + throw new Error(`Bad parse: missing end of group token: ${pattern}`) + } + const vals = pattern.slice(g + 1, end).split('|') + types.push(ts.factory.createUnionTypeNode(vals.map(v => ts.factory.createLiteralTypeNode(ts.factory.createStringLiteral(v))))) + + pattern = pattern.slice(end + 1) + } + + const head = ts.factory.createTemplateHead(strings.shift() ?? '') + const tail = ts.factory.createTemplateTail(strings.pop() ?? '') + if (isLiteral) { + if (types.length === 0) { + return ts.factory.createStringLiteral(head.text + tail.text) + } + + return types[0] + } + + const spans = types.slice(0, strings.length + 1).map((t, i) => { + const literal = i === strings.length ? tail : ts.factory.createTemplateMiddle(strings[i]) + + return ts.factory.createTemplateLiteralTypeSpan(t, literal) + }) + + return ts.factory.createTemplateLiteralType(head, spans) +} + +function normalizeName(name: string) { + return name.split(/[_\-\$.]/).map(s => s.charAt(0).toUpperCase().concat(s.slice(1))).join('') +} + +export function createSchemaGenerator(root: SchemaBase) { + function evalPointer(p: string) { + // ~0 for ~ + // ~1 for / + + const parts = p.split('/').map(s => s.replace(/~0/g, '~').replace(/~1/g, '/')) + const first = parts.shift() + if (first !== '#') { + throw new Error('Only document-relative pointers are supported') + } + + let c: any = root + while (parts.length > 0) { + const s = parts.shift()! + if (!c) { + throw new Error(`Attempted to index a falsy value at segment "${s}" in pointer "${p}"`) + } + c = c[s] + } + + return c + } + + function literal(val: Literal) { + if (val === null) { + return ts.factory.createLiteralTypeNode(ts.factory.createNull()) + } + + switch (typeof val) { + case 'number': + return ts.factory.createLiteralTypeNode(ts.factory.createNumericLiteral(val)) + case 'string': + return ts.factory.createLiteralTypeNode(ts.factory.createStringLiteral(val)) + case 'boolean': + return ts.factory.createLiteralTypeNode(val ? ts.factory.createTrue() : ts.factory.createFalse()) + } + } + + function renderString(schema: StringSchema) { + if (schema.enum) { + return ts.factory.createUnionTypeNode(schema.enum.filter(x => typeof x === 'string').map(literal)) + } + + return ts.factory.createTypeReferenceNode('string') + } + + function renderNumber(schema: NumberSchema) { + if (schema.enum) { + return ts.factory.createUnionTypeNode(schema.enum.filter(x => typeof x === 'number').map(literal)) + } + + return ts.factory.createTypeReferenceNode('number') + } + + function renderArray(schema: ArraySchema) { + // TODO: use `prefixItems` to create a tuple type + if (schema.prefixItems) { + console.log(schema) + } + + const items = schema.items !== undefined ? render(schema.items) : undefined + if (!items) { + throw new Error(`Not implemented`) + } + + return ts.factory.createArrayTypeNode(items) + } + + function renderObject(schema: ObjectSchema) { + const required = new Set(schema.required) + const members: ts.TypeElement[] = [] + + function isRequired(name: string, value: Schema) { + if (required.has(name)) { + return true + } + + // if (typeof value === 'object' && (value as any).default !== undefined) { + // return true + // } + + return false + } + + if (schema.properties) { + for (const [k, v] of Object.entries(schema.properties)) { + members.push(ts.factory.createPropertySignature( + [ts.factory.createModifier(ts.SyntaxKind.ReadonlyKeyword)], + quote(k), + !isRequired(k, v) ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) : undefined, + render(v), + )) + } + } + + if (schema.additionalProperties && !schema.properties) { + return ts.factory.createTypeReferenceNode('Record', [ + ts.factory.createTypeReferenceNode('string'), + render(schema.additionalProperties) + ]) + } + + if (schema.patternProperties) { + for (const [k, v] of Object.entries(schema.patternProperties)) { + const name = tryConvertRegExpPattern(k) + if (ts.isTemplateLiteralTypeNode(name)) { + members.push(ts.factory.createIndexSignature( + [ts.factory.createModifier(ts.SyntaxKind.ReadonlyKeyword)], + [ts.factory.createParameterDeclaration(undefined, undefined, 'name', undefined, name)], + ts.factory.createUnionTypeNode([ + render(v), + ts.factory.createTypeReferenceNode('undefined') + ]), + )) + } else if (ts.isStringLiteral(name)) { + members.push(ts.factory.createPropertySignature( + [ts.factory.createModifier(ts.SyntaxKind.ReadonlyKeyword)], + name, + !isRequired(k, v) ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) : undefined, + render(v), + )) + } else { + if (!ts.isUnionTypeNode(name)) { + throw new Error(`Unexpected type: ${name}`) + } + + for (const m of name.types) { + if (!ts.isLiteralTypeNode(m) || !ts.isStringLiteral(m.literal)) { + continue + } + members.push(ts.factory.createPropertySignature( + [ts.factory.createModifier(ts.SyntaxKind.ReadonlyKeyword)], + m.literal, + !isRequired(k, v) ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) : undefined, + render(v), + )) + } + } + } + } + + if (members.length === 0) { + return ts.factory.createTypeReferenceNode('any') + } + + return ts.factory.createTypeLiteralNode(members) + } + + const declarations = new Map() + + function render(schema: Schema): ts.TypeNode { + if (isRef(schema)) { + const name = normalizeName(schema.$ref.split('/').pop()!) + if (!declarations.has(name)) { + declarations.set(name, true as any) + + const v = render(evalPointer(schema.$ref)) + if (ts.isTypeLiteralNode(v)) { + const decl = ts.factory.createInterfaceDeclaration(undefined, name, undefined, undefined, v.members) + declarations.set(name, decl) + } else if (ts.isUnionTypeNode(v)) { + const decl = ts.factory.createTypeAliasDeclaration(undefined, name, undefined, v) + declarations.set(name, decl) + } else { + declarations.delete(name) + return v + } + } + + return ts.factory.createTypeReferenceNode(name) + } + + if (typeof schema === 'boolean') { + return ts.factory.createTypeReferenceNode(schema ? 'any' : 'never') + } + + if (isOneOf(schema)) { + return ts.factory.createUnionTypeNode(schema.oneOf.map(render).filter(isNonNullable)) + } + + // `anyOf` is treated the same as `oneOf` + if (isAnyOf(schema)) { + return ts.factory.createUnionTypeNode(schema.anyOf.map(render).filter(isNonNullable)) + } + + if (isAllOf(schema)) { + return ts.factory.createIntersectionTypeNode(schema.allOf.map(render).filter(isNonNullable)) + } + + if (Object.keys(schema).length === 0) { + return ts.factory.createTypeReferenceNode('any') + } + + if (Array.isArray(schema.type)) { + return ts.factory.createUnionTypeNode(schema.type.map(t => { + const s = { ...schema, type: t } + + return render(s) + })) + } + + if ((schema as any).not) { + return ts.factory.createTypeReferenceNode('any') + } + + if ((schema as any).properties && !schema.type) { + return render({ ...(schema as any), type: 'object' }) + } + + if ((schema as any).enum && !schema.type) { + return ts.factory.createUnionTypeNode((schema as any).enum.map(literal)) + } + + switch (schema.type) { + case 'null': + return literal(null) + case 'boolean': + return ts.factory.createTypeReferenceNode('boolean') // not entirely correct + case 'string': + return renderString(schema) + case 'number': + case 'integer': + return renderNumber(schema) + case 'array': + return renderArray(schema) + case 'object': + return renderObject(schema) + } + + throw new Error(`Unknown schema: ${JSON.stringify(schema)}`) + } + + function getDeclarations() { + return Array.from(declarations.values()) + } + + return { + render, + getDeclarations, + } +} + +async function fetchJson(url: string): Promise { + return fetch(url).then(r => r.json()) +} + + +export async function generateOpenApiV3() { + const doc = await fetchJson('https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.0/schema.json') + if (!isObjectSchema(doc)) { + throw new Error('Expected an object schema') + } + + const generator = createSchemaGenerator(doc) + const type = generator.render(doc) + if (!ts.isTypeLiteralNode(type)) { + throw new Error(`Unexpected return type node`) + } + + const decl = ts.factory.createInterfaceDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], + 'OpenApiModel', + undefined, + undefined, + type.members + ) + + const sf = ts.factory.createSourceFile( + [...generator.getDeclarations() as any, decl], + ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), + ts.NodeFlags.None + ) + + return printNodes(sf.statements, sf) +} + +export async function generateGitHubWebhooks() { + const doc = await fetchJson('https://unpkg.com/@octokit/webhooks-schemas@7.3.1/schema.json') + if (!doc.oneOf) { + throw new Error('Expected a `oneOf` field') + } + + const generator = createSchemaGenerator(doc) + + const members: ts.TypeElement[] = [] + for (const schema of doc.oneOf) { + if (!isRef(schema)) { + throw new Error(`Expected a reference`) + } + + const name = schema.$ref.split('/').pop()!.replace(/[_\$]event$/, '') + members.push(ts.factory.createPropertySignature( + undefined, + quote(name), + undefined, + generator.render(schema), + )) + } + + const finalDecl = ts.factory.createInterfaceDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], + 'WebhookEvent', + undefined, + undefined, + members + ) + + const sf = ts.factory.createSourceFile( + [...generator.getDeclarations() as any, finalDecl], + ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), + ts.NodeFlags.None + ) + + return printNodes(sf.statements, sf) +} + +interface EventObject { + readonly id: string + readonly object: 'event' + readonly api_version: string + readonly created: number + readonly type: string + // Info about the triggering request + readonly request?: any + readonly livemode: boolean + readonly data: { + readonly object: any + readonly previous_attributes?: any // Only included for events with type `*.updated` + } +} + +export async function generateStripeWebhooks() { + const doc = await fetchJson<{ components: { schemas: Record } }>('https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.sdk.json') + const events = Object.entries(doc.components.schemas).filter(([k, v]) => typeof v === 'object' && 'x-stripeEvent' in v) + + function createMember(name: string, type: ts.TypeNode, optional = false) { + return ts.factory.createPropertySignature( + undefined, + name, + optional ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) : undefined, + type, + ) + } + + const literalType = (literal: string | number) => { + const exp = typeof literal === 'string' + ? ts.factory.createStringLiteral(literal) + : ts.factory.createNumericLiteral(literal) + + return ts.factory.createLiteralTypeNode(exp) + } + + const anyType = ts.factory.createTypeReferenceNode('any') + const stringType = ts.factory.createTypeReferenceNode('string') + const numberType = ts.factory.createTypeReferenceNode('number') + const booleanType = ts.factory.createTypeReferenceNode('boolean') + + const eventObjBase = ts.factory.createInterfaceDeclaration( + undefined, + 'EventObjectBase', + undefined, + undefined, + [ + createMember('id', stringType), + createMember('created', numberType), + createMember('livemode', booleanType), + createMember('api_version', stringType), + createMember('request', anyType, true), + createMember('object', literalType('event')), + ] + ) + + const generator = createSchemaGenerator(doc as any) + + function generateEvent(type: string) { + const ref = generator.render({ $ref: `#/components/schemas/${type}` }) + const declName = `${normalizeName(type)}Event` + + return ts.factory.createInterfaceDeclaration( + undefined, + declName, + undefined, + [ts.factory.createHeritageClause( + ts.SyntaxKind.ExtendsKeyword, + [ts.factory.createExpressionWithTypeArguments(eventObjBase.name, undefined)] + )], + [ + createMember('type', literalType(type)), + // TODO: include `previous_attributes` for updated events + // TODO: create an object literal instead of a reference + createMember('data', ref) + ] + ) + } + + const eventDecls = events.map(([k]) => generateEvent(k)) + + const finalDecl = ts.factory.createTypeAliasDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], + 'EventObject', + undefined, + ts.factory.createUnionTypeNode(eventDecls.map(d => ts.factory.createTypeReferenceNode(d.name))) + ) + + const sf = ts.factory.createSourceFile( + [eventObjBase, ...generator.getDeclarations() as any, ...eventDecls, finalDecl], + ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), + ts.NodeFlags.None + ) + + return printNodes(sf.statements, sf) +} + + +// Stripe +// https://github.com/stripe/openapi +// +// Vendor extensions +// x-expandableFields +// x-expansionResources +// +// Webhooks +// "x-stripeEvent": { +// "type": "account.application.authorized" +// } +// +// https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.sdk.json \ No newline at end of file diff --git a/src/compiler/classifier.ts b/src/compiler/classifier.ts new file mode 100644 index 0000000..e363c7c --- /dev/null +++ b/src/compiler/classifier.ts @@ -0,0 +1,247 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { resolveProgramConfig } from './config' +import { getFs } from '../execution' +import { getWorkingDir } from '../workspaces' +import { Mutable, memoize } from '../utils' + +const semanticTokenTypes = [ + 'class', + 'enum', + 'interface', + 'namespace', + 'typeParameter', + 'type', + 'parameter', + 'variable', + 'enumMember', + 'property', + 'function', + 'member', +] as const + +type SemanticTokenType = (typeof semanticTokenTypes)[number] + +function getSemanticTokenType(n: number) { + return semanticTokenTypes[(n >> 8) - 1] as SemanticTokenType +} + +export interface ClassifiedSpan { + readonly start: number + readonly length: number + readonly syntacticType: ts.ClassificationType + readonly semanticType?: SemanticTokenType + readonly modifiers?: TokenModifiers +} + +interface TokenModifiers { + readonly declaration?: boolean + readonly static?: boolean + readonly async?: boolean + readonly readonly?: boolean + readonly defaultLibrary?: boolean + readonly local?: boolean +} + +function getModifiers(n: number): TokenModifiers { + const masked = n & 255 + + return { + declaration: !!(masked & (1 << 0)), + static: !!(masked & (1 << 1)), + async: !!(masked & (1 << 2)), + readonly: !!(masked & (1 << 3)), + defaultLibrary: !!(masked & (1 << 4)), + local: !!(masked & (1 << 5)), + } +} + +async function createLanguageServiceHost() { + const fs = getFs() + const config = await resolveProgramConfig() + + const copy = { ...config.tsc.cmd.options, lib: ['lib.es5.d.ts'], types: [] } + + const files: Record = {} + for (const f of config.tsc.cmd.fileNames) { + files[f] = { version: 0 } + } + + const servicesHost: ts.LanguageServiceHost = { + getScriptFileNames: () => config.tsc.cmd.fileNames, + getScriptVersion: fileName => files[fileName]?.version.toString(), + getScriptSnapshot: fileName => { + if (!fs.fileExistsSync(fileName)) { + return undefined + } + + return ts.ScriptSnapshot.fromString(fs.readFileSync(fileName).toString()) + }, + getCurrentDirectory: () => process.cwd(), + getCompilationSettings: () => copy, + getDefaultLibFileName: options => ts.getDefaultLibFilePath(options), + fileExists: ts.sys.fileExists, + readFile: ts.sys.readFile, + readDirectory: ts.sys.readDirectory, + directoryExists: ts.sys.directoryExists, + getDirectories: ts.sys.getDirectories, + } + + return servicesHost +} + +// function enhanceSpans(typeChecker: ts.TypeChecker, sf: ts.SourceFile, spans: ClassifiedSpan[]) { +// function findNode(pos: number, end: number, parent: ts.Node = sf): ts.Node | undefined { +// return parent.forEachChild(n => { +// if (n.pos === pos - n.getLeadingTriviaWidth() && n.end === end) { +// return n +// } + +// return findNode(pos, end, n) +// }) +// } + +// for (const s of spans) { +// if (s.semanticType === 'function' || s.semanticType === 'type') continue + +// const node = findNode(s.start, s.start + s.length) +// if (!node) continue + +// switch (node.kind) { +// case ts.SyntaxKind.BooleanKeyword: +// case ts.SyntaxKind.StringKeyword: +// case ts.SyntaxKind.NumberKeyword: +// ;(s as Mutable).semanticType = 'type' +// continue +// } + +// if (node.kind === ts.SyntaxKind.UndefinedKeyword || node.kind === ts.SyntaxKind.NullKeyword) { +// const isTypeNode = ts.findAncestor(node, p => ts.isTypeNode(p)) +// if (isTypeNode) { +// ;(s as Mutable).semanticType = 'type' +// } + +// continue +// } + +// if (ts.isIdentifier(node) && (node.text === 'undefined' || node.text === 'null')) { +// ;(s as Mutable).semanticType = undefined +// ;(s as Mutable).syntacticType = ts.ClassificationType.keyword +// continue +// } + +// const type = typeChecker.getTypeAtLocation(node) +// if (type.getCallSignatures().length > 0) { +// ;(s as Mutable).semanticType = 'function' +// continue +// } +// } + +// return spans +// } + +export const getLanguageService = memoize(async () => { + const host = await createLanguageServiceHost() + + return ts.createLanguageService(host) +}) + +function getClassifiedSpans(service: ts.LanguageService, sf: ts.SourceFile, span = ts.createTextSpan(0, sf.text.length)) { + const spans = new Map() + + function getSpan(start: number, length: number): ClassifiedSpan { + const key = `${start}:${length}` + if (spans.has(key)) { + return spans.get(key)! + } + + const s: ClassifiedSpan = { + start, + length, + syntacticType: ts.ClassificationType.text, + } + spans.set(key, s) + + return s + } + + const syntactic = service.getEncodedSyntacticClassifications( + sf.fileName, + span, + ) + + for (let i = 0; i < syntactic.spans.length; i += 3) { + const s = getSpan(syntactic.spans[i], syntactic.spans[i+1]) + ;(s as Mutable).syntacticType = syntactic.spans[i+2] + } + + const semantic = service.getEncodedSemanticClassifications( + sf.fileName, + span, + ts.SemanticClassificationFormat.TwentyTwenty, + ) + + for (let i = 0; i < semantic.spans.length; i += 3) { + const s = getSpan(semantic.spans[i], semantic.spans[i+1]) + ;(s as Mutable).modifiers = getModifiers(semantic.spans[i+2]) + ;(s as Mutable).semanticType = getSemanticTokenType(semantic.spans[i+2]) + } + + return [...spans.values()].sort((a, b) => a.start - b.start) +} + +interface Position { + readonly line: number + readonly column: number +} + +interface Range { + readonly start: Position | number + readonly end: Position | number +} + +export async function getClassifications(fileName: string, range?: Range) { + const s = await getLanguageService() + const p = s.getProgram() + if (!p) { + throw new Error(`No program found`) + } + + const rp = path.resolve(getWorkingDir(), fileName) + const sf = p.getSourceFile(rp) + if (!sf) { + throw new Error(`Missing source file: ${rp}`) + } + + const lines = sf.getLineStarts() + + function getOffset(p: Position | number) { + if (typeof p === 'number') { + return p + } + + const line = Math.min(p.line, lines.length - 1) + const eol = line === lines.length - 1 + ? sf!.getEnd()-1 + : lines[line+1]-1 + + return Math.min(p.column + lines[line], eol) + } + + if (range) { + const start = getOffset(range.start) + const end = getOffset(range.end) + + return { + sourceFile: sf, + spans: getClassifiedSpans(s, sf, ts.createTextSpan(start, end - start)), + getOffset, + } + } + + return { + sourceFile: sf, + spans: getClassifiedSpans(s, sf), + getOffset, + } +} diff --git a/src/compiler/config.ts b/src/compiler/config.ts new file mode 100644 index 0000000..963d1e1 --- /dev/null +++ b/src/compiler/config.ts @@ -0,0 +1,638 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { Fs, JsonFs } from '../system' +import { CompilerOptions } from './host' +import { glob } from '../utils/glob' +import { getLogger, runTask } from '..' +import { getBuildTargetOrThrow, getFs, getSelfPathOrThrow, isSelfSea } from '../execution' +import { PackageJson, getPreviousPkg } from '../pm/packageJson' +import { getProgramFs } from '../artifacts' +import { getWorkingDir } from '../workspaces' +import { getHash, makeRelative, memoize, resolveRelative, throwIfNotFileNotFoundError } from '../utils' +import { readKeySync } from '../cli/config' + +interface ParsedConfig { + readonly cmd: Pick + readonly files: string[] + readonly rootDir: string + readonly sourceHash: string + readonly include?: string[] + readonly exclude?: string[] + readonly previousOptions?: ts.CompilerOptions +} + +// TODO: +// substitute `${configDir}` in `tsconfig.json` +// https://devblogs.microsoft.com/typescript/announcing-typescript-5-5-beta/#the-configdir-template-variable-for-configuration-files + +// "moduleDetection": "force", +// "moduleResolution": "bundler", +// "allowImportingTsExtensions": true, +// "verbatimModuleSyntax": true, +// "noEmit": true, + +function getDefaultTsConfig(targetFiles: string[]) { + return { + include: targetFiles, + compilerOptions: { + target: 'ES2022', + module: 'NodeNext', + moduleResolution: 'NodeNext', + resolveJsonModule: true, + sourceMap: true, + esModuleInterop: true, + strict: true, + skipLibCheck: true, + alwaysStrict: true, + forceConsistentCasingInFileNames: true, + }, + } +} + +function getTsConfigFromText(configText: string | void, fileName: string, targetFiles?: string[]) { + if (!configText) { + getLogger().debug(`No tsconfig.json, using default`) + + return getDefaultTsConfig(targetFiles ?? ['*']) + } + const parseResult = ts.parseConfigFileTextToJson(fileName, configText) + if (parseResult.error) { + // TODO: output the error better, does `typescript` have a way to do this? (yes it does) + throw Object.assign(new Error('Failed to parse "tsconfig.json"'), parseResult.error) + } + + return parseResult.config +} + +// Settings that affect the interpretation of source code need to be directly +// supported by us, otherwise things just break. Best to fail early. +const notSupportedOptions = [ + 'paths', + 'baseUrl', + 'rootDirs', +] + +// These settings don't affect how the user might have written their program +// So we can still compile, the results just might not be what the user expects +// Either because we override settings, or because we simply ignore them +const notSupportedWarningOptions = [ + 'noEmit', + 'plugins', + 'outFile', + 'importHelpers', +] + +// TODO: does not handle the "extends" target changing +async function getTsConfig( + fs: Fs, + workingDirectory: string, + targetFiles?: string[], + fileName = path.resolve(workingDirectory, 'tsconfig.json'), + sys = ts.sys, +): Promise { + const [text, previousConfig] = await Promise.all([ + getFs().readFile(fileName, 'utf-8').catch(throwIfNotFileNotFoundError), + getResolvedTsConfig() + ]) + + const sourceHash = text ? getHash(text) : '' + + function parse() { + const config = getTsConfigFromText(text, fileName, targetFiles) + const cmd = ts.parseJsonConfigFileContent(config, sys, workingDirectory, undefined, fileName) + if (cmd.errors.length > 0) { + // TODO: output the error better, does `typescript` have a way to do this? (yes it does) + // ts.formatDiagnostic(d, {}) + throw Object.assign(new Error('Failed to parse "tsconfig.json"'), cmd.errors) + } + + cmd.options.composite ??= true + cmd.options.alwaysStrict ??= true + cmd.options.sourceMap ??= true + cmd.options.skipLibCheck ??= true + + if (cmd.options.composite === false) { + throw new Error('Programs cannot be compiled with `composite` set to `false`') + } + + for (const k of notSupportedWarningOptions) { + // TODO: warning + } + + const notSupported = notSupportedOptions.filter(k => !!cmd.options[k]) + if (notSupported.length > 0) { + throw new Error(`The following tsconfig.json options are not supported (yet): ${notSupported.join(', ')}`) + } + + const rootDir = cmd.options.rootDir + ? path.resolve(workingDirectory, cmd.options.rootDir) + : workingDirectory + + cmd.options.rootDir = rootDir + + const hasTsx = !!cmd.fileNames.find(f => f.endsWith('.tsx')) + if (hasTsx && cmd.options.jsx === undefined) { + cmd.options.jsx = ts.JsxEmit.ReactJSX + } + + return cmd + } + + const isCached = previousConfig?.sourceHash === sourceHash + const cmd: ParsedConfig['cmd'] = isCached + ? { options: previousConfig.options, fileNames: [], raw: { include: previousConfig.include, exclude: previousConfig.exclude } } + : parse() + + // TODO: fail early if a file is outside of the root directory + const exclude = cmd.raw?.exclude ?? ['node_modules'] + const files = await runTask('glob', 'tsc-files', () => glob(fs, workingDirectory, cmd.raw?.include ?? ['*'], exclude), 1) + if (isCached) { + cmd.fileNames = files + } + + // `include` and `exclude` are only for caching + return { + cmd, + files, + rootDir: cmd.options.rootDir!, + sourceHash, + include: cmd.raw?.include, + exclude: cmd.raw?.exclude, + previousOptions: previousConfig?.options, + } +} + +function replaceFileExtension(opt: ts.CompilerOptions, fileName: string) { + return opt.jsx === ts.JsxEmit.Preserve + ? fileName.replace(/\.t(sx?)$/, `.j$1`) + : fileName.replace(/\.tsx?$/, `.js`) +} + +export function getOutputFilename(rootDir: string, opt: ts.CompilerOptions, fileName: string) { + const relPath = path.relative(rootDir, fileName) + if (relPath.startsWith('..')) { + throw new Error(`File "${fileName}" is outside of the root directory`) + } + + const resolved = path.resolve(opt.outDir ?? rootDir, relPath) + + return replaceFileExtension(opt, resolved) +} + +export function getInputFilename(rootDir: string, opt: ts.CompilerOptions, fileName: string) { + const outDir = opt.outDir ?? rootDir + if (!fileName.startsWith(outDir) || !fileName.endsWith('.js')) { + return fileName + } + + const resolved = path.resolve(rootDir, path.relative(outDir, fileName)) + + return resolved.replace(/\.js$/, '.ts') // XXX: incorrect, source could have been `.tsx` +} + +export interface ResolvedProgramConfig { + readonly tsc: ParsedConfig + readonly csc: CompilerOptions + readonly pkg?: PackageJson + readonly compiledEntrypoints?: string[] +} + +// Discovers all potential entrypoints to a package via: +// * `bin` +// * `main` +// * `module` +// * `exports` +function getEntrypointPatterns(pkg: PackageJson) { + const entrypoints: string[] = [] + + if (pkg.main) { + entrypoints.push(pkg.main) + } + + if (pkg.module) { + entrypoints.push(pkg.module) + } + + if (pkg.bin) { + if (typeof pkg.bin === 'string') { + entrypoints.push(pkg.bin) + } else { + for (const v of Object.values(pkg.bin)) { + entrypoints.push(v) + } + } + } + + if (pkg.exports) { + // TODO: handle all cases + if (typeof pkg.exports === 'string') { + entrypoints.push(pkg.exports) + } else if (typeof pkg.exports === 'object') { + for (const [k, v] of Object.entries(pkg.exports)) { + if (!k.startsWith('.') || typeof v !== 'string') { + continue + } + + entrypoints.push(v) + } + } + } + + return entrypoints +} + +// Transforms all entrypoints to use the expected output file +// This allows you to write something like "src/cli/index.ts" in `package.json` instead of the output file +function resolvePackageEntrypoints(pkg: PackageJson, dir: string, rootDir: string, opt: ts.CompilerOptions) { + const res = { ...pkg } + const compiledEntrypoints = new Set() + + function resolve(p: string) { + const resolved = resolveRelative(dir, p) + if (opt.outDir && resolved.startsWith(opt.outDir)) { + return p + } + + const sourceFile = path.resolve(dir, p) + const outfile = getOutputFilename(rootDir, opt, sourceFile) + if (sourceFile !== outfile) { + compiledEntrypoints.add(sourceFile) + } + + return `./${makeRelative(dir, outfile)}` + } + + if (res.main) { + res.main = resolve(res.main) + } + + if (res.module) { + res.module = resolve(res.module) + } + + if (res.bin) { + if (typeof res.bin === 'string') { + res.bin = resolve(res.bin) + } else { + // MUTATES + for (const [k, v] of Object.entries(res.bin)) { + res.bin[k] = resolve(v) + } + } + } + + if (res.exports) { + // TODO: handle all cases + if (typeof res.exports === 'string') { + res.exports = resolve(res.exports) + } else if (typeof res.exports === 'object') { + for (const [k, v] of Object.entries(res.exports)) { + if (!k.startsWith('.') || typeof v !== 'string') { + continue + } + + res.exports[k] = resolve(v) + } + } + } + + return { + packageJson: res, + compiledEntrypoints: [...compiledEntrypoints].map(x => makeRelative(dir, x)), + } +} + +// TODO: this can be made more efficient by using `parsed.files` +async function resolveEntrypoints(dir: string, patterns: string[], parsed: ParsedConfig) { + const resolvedPatterns = patterns.map(p => { + const rel = path.relative( + dir, + getInputFilename(parsed.rootDir, parsed.cmd.options, path.resolve(dir, p)) + ) + + return rel.replace(/\/([^\/*]*\*[^\/*]*\/?)/, '/**/$1') + }) + + // Match `.tsx` as well + if (parsed.cmd.options.jsx !== undefined) { + for (const p of resolvedPatterns) { + const alt = p.replace(/\.ts$/, '.tsx') + if (p !== alt) { + resolvedPatterns.push(alt) + } + } + } + + return await glob(getFs(), dir, resolvedPatterns, ['node_modules']) +} + +async function resolvePackage(pkg: PackageJson, dir: string, parsed: ParsedConfig) { + const compiled = resolvePackageEntrypoints(pkg, dir, parsed.rootDir, parsed.cmd.options) + + return { pkg: compiled.packageJson, compiledEntrypoints: compiled.compiledEntrypoints } + // const patterns = getEntrypointPatterns(pkg) + // if (patterns.length === 0) { + // return { pkg: compiled } + // } + + // const entrypoints = await resolveEntrypoints(dir, patterns, parsed) + + // return { pkg: compiled, entrypoints } +} + +// Creates a package.json file for one-off scripts/experiments/etc. +// This isn't exposed to the user +function createSyntheticPackage(opt?: CompilerOptions, targetFiles?: string[]) { + return { + pkg: { + "synapse": opt?.deployTarget ? { + "config": { + "target": opt.deployTarget, + }, + } : undefined + } as PackageJson, + compiledEntrypoints: undefined as string[] | undefined + } +} + +// Merged left to right (lower -> higher precedence) +function mergeConfigs(...configs: (T | undefined)[]): Partial { + const res: Partial = {} + for (const c of configs) { + if (!c) continue + + for (const [k, v] of Object.entries(c)) { + if (v !== undefined) { + res[k as keyof T] = v as any + } + } + } + + return res +} + +export async function resolveProgramConfig(opt?: CompilerOptions, targetFiles?: string[], fs = getFs()): Promise { + patchTsSys() + + const bt = getBuildTargetOrThrow() + const [parsed, pkg, previousPkg] = await Promise.all([ + runTask('resolve', 'tsconfig', () => getTsConfig(fs, bt.workingDirectory, targetFiles), 5), + getFs().readFile(path.resolve(bt.workingDirectory, 'package.json'), 'utf-8').then(JSON.parse).catch(throwIfNotFileNotFoundError), + getPreviousPkg() + ]) + + const resolvedPkg = pkg !== undefined + ? await runTask('resolve', 'pkg', () => resolvePackage(pkg, bt.workingDirectory, parsed), 5) + : createSyntheticPackage(opt, targetFiles) + + const deployTarget = opt?.deployTarget ?? (previousPkg?.synapse?.config?.target ?? 'local') + if (deployTarget) { + const p = resolvedPkg.pkg as any + const s = p.synapse ??= {} + s.config ??= {} + s.config.target ??= deployTarget + } + + const pkgConfig: CompilerOptions = { + ...resolvedPkg?.pkg.synapse?.config, + deployTarget: resolvedPkg?.pkg.synapse?.config?.target, + } + + const cscOpt: CompilerOptions = { + includeJs: true, + generateExports: true, + excludeProviderTypes: true, + ...mergeConfigs(pkgConfig, opt), + } + + if (cscOpt.stripInternal) { + parsed.cmd.options.stripInternal = true + } + + async function getTypeDirs() { + const typesDir = path.resolve(bt.workingDirectory, 'node_modules', '@types') + + try { + const dirs = (await fs.readDirectory(typesDir)).filter(f => f.type === 'directory') + + return dirs.map(f => f.name).filter(n => n !== 'synapse-providers') + } catch (e) { + throwIfNotFileNotFoundError(e) + + return parsed.cmd.options.types + } + } + + parsed.cmd.options.types ??= cscOpt.excludeProviderTypes ? await getTypeDirs() : undefined + parsed.cmd.options.declaration ??= !!cscOpt.sharedLib ? true : undefined + + // By default, we'll only include the bare minimum libs to speed-up program init time + // Normally `tsc` would include `lib.dom.d.ts` but that file is pretty big + // + // We can use `noLib` if we use our own type checker and/or provide our own type defs + const maybeNeedsDeclaration = parsed.cmd.options.declaration || (resolvedPkg.compiledEntrypoints?.length ?? 0) > 0 + if (!maybeNeedsDeclaration) { + parsed.cmd.options.noLib = true + getLogger().log('Using "noLib" for compilation') + } else { + parsed.cmd.options.lib ??= libFromTarget(ts.ScriptTarget.ES5) + } + + const config: ResolvedProgramConfig = { + tsc: parsed, + csc: cscOpt, + pkg: resolvedPkg.pkg, + compiledEntrypoints: resolvedPkg.compiledEntrypoints, + } + + // Not awaited intentionally + saveResolvedConfig(bt.workingDirectory, config) + + getLogger().emitResolveConfigEvent({ config }) + + return config +} + +const tsOptionsFileName = `[#compile/config]__tsoptions__.json` +const tsOptionsPathKeys = ['baseUrl', 'configFilePath', 'rootDir', 'outDir'] // This isn't all of them. Update as-needed. + +interface ResolvedTsConfig { + readonly version: string // `tsc` version + readonly options: ts.CompilerOptions + readonly sourceHash: string + readonly include?: string[] + readonly exclude?: string[] +} + +function makeObjRelative>(from: string, obj: T, keys: (keyof T)[]): T { + const set = new Set(keys) + const copied = { ...obj } as any + + for (const [k, v] of Object.entries(copied)) { + if (!set.has(k)) continue + + if (typeof v === 'string') { + copied[k] = path.relative(from, v) + } else if (Array.isArray(v)) { + copied[k] = v.map(x => path.relative(from, x)) + } + } + + return copied +} + +// in-place is OK +function resolveObjRelative>(from: string, obj: T, keys: (keyof T)[]): T { + const set = new Set(keys) + const mut = obj as any + + for (const [k, v] of Object.entries(obj)) { + if (!set.has(k)) continue + + if (typeof v === 'string') { + mut[k] = path.resolve(from, v) + } else if (Array.isArray(v)) { + mut[k] = v.map(x => path.resolve(from, x)) + } + } + + return mut +} + + +async function saveResolvedConfig(workingDir: string, config: ResolvedProgramConfig) { + const resolved: ResolvedTsConfig = { + version: ts.version, + options: makeObjRelative(workingDir, config.tsc.cmd.options, tsOptionsPathKeys), + sourceHash: config.tsc.sourceHash, + include: config.tsc.include, + exclude: config.tsc.exclude, + } + + await getProgramFs().writeJson(tsOptionsFileName, resolved) +} + +export async function getResolvedTsConfig(fs: Pick = getProgramFs(), workingDir = getWorkingDir()): Promise { + const unresolved: ResolvedTsConfig | undefined = await fs.readJson(tsOptionsFileName).catch(throwIfNotFileNotFoundError) + if (!unresolved) { + return + } + + return { + version: unresolved.version, + options: resolveObjRelative(workingDir, unresolved.options, tsOptionsPathKeys), + sourceHash: unresolved.sourceHash, + include: unresolved.include, + exclude: unresolved.exclude, + } +} + +// Only returns different keys +function shallowDiff>(a: T, b: T, keys?: Set): Set<(keyof T)> { + const diff = new Set() + if (!keys) { + keys = new Set([...Object.keys(a), ...Object.keys(b)]) + } + + for (const k of keys) { + const valA = a[k] + const valB = b[k] + const type = typeof valA + if (type !== typeof valB) continue + + if (type !== 'object') { + if (valA !== valB) { + diff.add(k) + } + } else if (Array.isArray(valA)) { + if (valA.length !== valB.length) continue + + valA.sort() + valB.sort() + + for (let i = 0; i < valA.length; i++) { + if (valA[i] !== valB[i]) { + diff.add(k) + break + } + } + } + + // TODO: objects + } + + return diff +} + +// TODO: there's more keys that need to be added +const invalidationKeys = new Set([ + 'target', + 'module', + 'outDir', + 'jsx', + 'declaration', + 'stripInternal', + 'declarationMap', +]) + +// TODO: some changes don't need recompilation +// for example disabling `declaration` can be handled entirely at the emit phase +export function shouldInvalidateCompiledFiles(tsc: ParsedConfig) { + const prev = tsc.previousOptions + if (!prev) { + return false + } + + const cur = tsc.cmd.options + const changed = shallowDiff(cur, prev, new Set(invalidationKeys)) + if (changed.size === 0) { + return false + } + + getLogger().log('Changed tsconfig keys', changed) + return true +} + +// We exclude the DOM type def automatically unless explicitly added in `lib` +function libFromTarget(target: ts.ScriptTarget) { + switch (target) { + case ts.ScriptTarget.ESNext: + return ['lib.esnext.d.ts'] + case ts.ScriptTarget.ES2022: + return ['lib.es2022.d.ts'] + case ts.ScriptTarget.ES2021: + return ['lib.es2021.d.ts'] + case ts.ScriptTarget.ES2020: + return ['lib.es2020.d.ts'] + case ts.ScriptTarget.ES2019: + return ['lib.es2019.d.ts'] + case ts.ScriptTarget.ES2018: + return ['lib.es2018.d.ts'] + case ts.ScriptTarget.ES2017: + return ['lib.es2017.d.ts'] + case ts.ScriptTarget.ES2016: + return ['lib.es2016.d.ts'] + case ts.ScriptTarget.ES2015: // same thing as ES6 + return ['lib.es2015.d.ts'] + case ts.ScriptTarget.ES5: + return ['lib.es5.d.ts'] + } +} + +const patchTsSys = memoize(() => { + // The filepath returned by `getExecutingFilePath` doesn't need to exist + const libDir = readKeySync('typescript.libDir') + if (typeof libDir === 'string') { + ts.sys.getExecutingFilePath = () => path.resolve(libDir, 'cli.js') + + return + } + + if (!isSelfSea()) { + return + } + + const selfPath = getSelfPathOrThrow() + ts.sys.getExecutingFilePath = () => path.resolve(selfPath, '..', '..', 'dist', 'cli.js') +}) + diff --git a/src/compiler/declarations.ts b/src/compiler/declarations.ts new file mode 100644 index 0000000..4995ac8 --- /dev/null +++ b/src/compiler/declarations.ts @@ -0,0 +1,234 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { Fs, SyncFs } from '../system' +import { getWorkingDir } from '../workspaces' +import { SourceMapHost } from '../static-solver/utils' +import { SourceMapV3, mergeSourcemaps } from '../runtime/sourceMaps' +import { AmbientDeclarationFileResult, isRelativeSpecifier, makeRelative, memoize, resolveRelative, toAmbientDeclarationFile } from '../utils' + + +function getLongestCommonPrefix(paths: string[]) { + if (paths.length === 0) { + return '' + } + + const res: string[] = [] + const split = paths.map(p => p.split(path.sep)).sort((a, b) => a.length - b.length) + outer: for (let i = 0; i < split[0].length; i++) { + for (let j = 1; j < split.length; j++) { + if (split[j][i] !== split[j - 1][i]) { + break outer + } + } + + res.push(split[0][i]) + } + + return res.join(path.sep) +} + +export type DeclarationFileHost = ReturnType +export function createDeclarationFileHost(fs: Fs & SyncFs, sourcemapHost: SourceMapHost) { + // Maps filename -> module id e.g. `/foo/qaz/bar.d.ts` -> `foo:bar` + const ambientDeclFileMap = new Map() + const sourcemaps = new Map() + const declarations = new Map() + const declarationsInversed = new Map() + const references = new Map>() + + function addSourcemap(declarationFile: string, text: string) { + sourcemaps.set(declarationFile, text) + } + + function addDeclaration(source: string, fileName: string, text: string) { + declarations.set(source, { fileName, text }) + } + + function setBinding(fileName: string, moduleId: string) { + // XXX: windows hack + if (process.platform === 'win32') { + fileName = fileName.replaceAll('\\', '/') + } + declarationsInversed.set(fileName, moduleId) + } + + function resolve(spec: string, importer: string) { + const p = resolveRelative(path.dirname(importer), spec) + const candidates = [ + `${p}.ts`, + `${p}.d.ts`, + resolveRelative(p, 'index.ts'), + resolveRelative(p, 'index.d.ts') + ] + + for (const c of candidates) { + if (ambientDeclFileMap.has(c)) { + return c + } + + if (declarationsInversed.has(c)) { + return c + } + + if (fs.fileExistsSync(c)) { + return c + } + } + + throw new Error(`Failed to resolve "${spec}" [importer: ${importer}]`) + } + + function getDeclarationSourceFile(fileName: string) { + const contents = fs.readFileSync(fileName, 'utf-8') + + return ts.createSourceFile(fileName, contents, ts.ScriptTarget.ES2022) + } + + const deps = new Map>() + function getRelativeDependencies(fileName: string) { + if (deps.has(fileName)) { + return deps.get(fileName)! + } + + const d = new Set() + deps.set(fileName, d) + + const sf = getDeclarationSourceFile(fileName) + for (const s of sf.statements) { + if (!ts.isImportDeclaration(s) && !ts.isExportDeclaration(s)) { + continue + } + + const spec = (s.moduleSpecifier as ts.StringLiteral | undefined)?.text + if (!spec || !isRelativeSpecifier(spec)) { + continue + } + + const resolved = resolve(spec, fileName) + if (declarationsInversed.has(resolved)) { + continue + } + + d.add(resolved) + } + + return d + } + + function addRef(moduleId: string, ref: string) { + const s = references.get(moduleId) ?? new Set() + references.set(moduleId, s) + + if (references.has(ref)) { + return + } + + s.add(ref) + const deps = getRelativeDependencies(ref) + for (const d of deps) { + addRef(moduleId, d) + } + } + + const getOutputPrefix = memoize(() => getLongestCommonPrefix([...declarationsInversed.keys()])) + + function transform(declFile: ts.SourceFile, moduleId: string): AmbientDeclarationFileResult { + const key = declFile.fileName + if (ambientDeclFileMap.has(key)) { + return ambientDeclFileMap.get(key)! + } + + const result = toAmbientDeclarationFile(moduleId, declFile, sourcemapHost, (spec, importer) => { + const fileName = resolve(spec, importer) + if (declarationsInversed.has(fileName)) { + return declarationsInversed.get(fileName)! + } + + addRef(moduleId, fileName) + + return spec + }) + + ambientDeclFileMap.set(key, result) + + return result + } + + function getDeclaration(source: string) { + const decl = declarations.get(source) + if (!decl) { + throw new Error(`Missing declaration file for: ${source}`) + } + + return decl + } + + function getSourcemap(fileName: string) { + const text = sourcemaps.get(fileName) + if (!text) { + throw new Error(`Missing sourcemap for declaration file: ${fileName}`) + } + + return text + } + + function mergeMapping(source: string, sourcemap: SourceMapV3) { + const decl = getDeclaration(source) + const text = getSourcemap(decl.fileName) + const originalMap: SourceMapV3 = JSON.parse(text) + + return mergeSourcemaps(originalMap, sourcemap) + } + + function finalizeResult(source: string, result: AmbientDeclarationFileResult) { + const dest = makeRelative( + getOutputPrefix(), + path.resolve(getWorkingDir(), getDeclaration(source).fileName) + ) + + const merged = mergeMapping(source, result.sourcemap) + merged.file = path.basename(dest) + merged.sources[0] = source + + const mappedRefs = [...references.get(result.id) ?? []].map(r => { + const f = makeRelative(getOutputPrefix(), r) + if (f.startsWith('..')) { + throw new Error(`Cannot reference file outside of the root module directory: ${r}`) + } + return [f, makeRelative(getWorkingDir(), r)] + }) + + return { + name: dest, + text: result.text, + sourcemap: JSON.stringify(merged), + references: Object.fromEntries(mappedRefs), + } + } + + async function createAmbientDeclarations() { + const files: Record = {} + const sorted = Array.from(ambientDeclFileMap.entries()).sort((a, b) => a[1].id.localeCompare(b[1].id)) + for (const [k, v] of sorted) { + files[v.id] = finalizeResult(k, v) + } + + return files + } + + function transformModuleBinding(source: string, moduleBinding: string) { + const decl = getDeclaration(source) + const sf = ts.createSourceFile(decl.fileName, decl.text, ts.ScriptTarget.ES2022) + const result = transform(sf, moduleBinding) + + return finalizeResult(source, result) + } + + return { + setBinding, + addSourcemap, + addDeclaration, + emitAmbientDeclarations: createAmbientDeclarations, + transformModuleBinding, + } +} \ No newline at end of file diff --git a/src/compiler/diagnostics.ts b/src/compiler/diagnostics.ts new file mode 100644 index 0000000..2c01a01 --- /dev/null +++ b/src/compiler/diagnostics.ts @@ -0,0 +1,286 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { getFs } from '../execution' +import { RenderableError, colorize, dim, printLine, stripAnsi } from '../cli/ui' +import { ClassifiedSpan, getClassifications, getLanguageService } from './classifier' +import { getLogger } from '..' +import { AsyncResource } from 'async_hooks' +import { getWorkingDir } from '../workspaces' + +interface SourceLocation { + readonly line: number // 1-indexed + readonly column: number // 1-indexed + readonly fileName: string + + // For highlighting/marking text ranges + readonly length?: number + + // Text describing the problem/solution/reason etc. + readonly annotation?: string +} + +interface Diagnostic { + // readonly code: number + readonly severity: 'warning' | 'error' + readonly message: string + readonly sources?: SourceLocation[] +} + +function getNodeLocation(node: ts.Node, annotation?: string): SourceLocation { + const sf = node.getSourceFile() + if (!sf) { + return { + line: 0, + column: 0, + length: node.end - node.pos, + annotation, + fileName: 'unknown', + } + } + + const pos = node.pos + node.getLeadingTriviaWidth(sf) + const lc = sf.getLineAndCharacterOfPosition(pos) + + return { + annotation, + line: lc.line + 1, + column: lc.character + 1, + fileName: sf.fileName, + length: node.end - pos, + } +} + +interface FormatDiagnosticOptions { + readonly workingDir?: string + readonly showSource?: boolean +} + +interface FormatSnippetOptions { + context?: number + maxWidth?: number + colorizer?: (text: string, line: number, col: number) => string +} + +function getSourceSnippet(source: SourceLocation, text: string, opt: FormatSnippetOptions = {}) { + const { + context = 1, + maxWidth = Math.min(80, process.stdout.columns), + colorizer + } = opt + + const lines = text.split(/\r?\n/) + const lineStart = Math.max(1, source.line - context) + const selection = lines.slice(lineStart - 1, source.line) + if (selection.length === 0) { + return [] + } + + const truncationSymbol = '...' + const gutterBorder= `| ` + const gutterWidth = String(source.line).length + gutterBorder.length + + // TODO: prefer discarding whitespace instead of centering + const screenOffset = (source.column - 1) >= maxWidth + ? (source.column - 1) - Math.floor(maxWidth / 2) + : 0 + + function formatLine(text: string, lineNumber: number) { + const lineNumberText = String(lineNumber).padStart(gutterWidth - gutterBorder.length, ' ') + const gutter = lineNumber === source.line + ? `${lineNumberText}${dim(gutterBorder)}` + : dim(`${lineNumberText}${gutterBorder}`) + + function withColor(text: string) { + if (!colorizer) return text + + return colorizer(text, lineNumber - 1, screenOffset) + } + + // TODO: don't add truncation marker if it's just whitespace + if (text.length + gutterWidth >= maxWidth) { + const truncated = text.slice(0, maxWidth - (gutterWidth + truncationSymbol.length)) + + return `${gutter}${withColor(truncated)}${dim(truncationSymbol)}` + } + + return `${gutter}${withColor(text)}` + } + + const formatted = selection + .map(t => t.slice(screenOffset)) + .map((t, i) => formatLine(t, i + lineStart)) + + function makeFooter() { + const len = source.length ?? 1 + const squiggles = colorize('brightRed', '~'.repeat(len).padStart(gutterWidth + (source.column - screenOffset) + len-1, ' ')) + if (source.annotation) { + return `${squiggles} ${colorize('brightRed', source.annotation)}` + } + + return squiggles + } + + return [ + ...formatted, + makeFooter(), + ] +} + +const controlKeywords = [ + 'await', + 'import', + 'export', + 'from', + 'switch', + 'case', + 'return', + 'break', + 'continue', + 'if', + 'else', + 'throw', + 'for', + 'default', +] + +// TODO: primitive types are treated as keywords +const primitives = ['number', 'string', 'boolean', 'symbol', 'bigint', 'null'] + +function colorizeSpan(text: string, span: ClassifiedSpan): string { + if (span.semanticType) { + switch (span.semanticType) { + case 'function': + return colorize('paleYellow', text) + + case 'type': + case 'typeParameter': + case 'class': + return colorize('brightGreen', text) + + case 'member': + case 'property': + case 'parameter': + case 'variable': + if (span.modifiers?.readonly) { + return colorize('brightCyan', text) + } + return colorize('paleCyan', text) + } + } + + switch (span.syntacticType) { + case ts.ClassificationType.comment: + return colorize('commentGreen', text) + + case ts.ClassificationType.keyword: + if (controlKeywords.includes(text)) { + return colorize('brightPurple', text) + } + + return colorize('brightBlue', text) + + case ts.ClassificationType.stringLiteral: + if (text.startsWith('/') && text.endsWith('/')) { // TODO: flags + return colorize('red', text) // regexp literal + } + return colorize('orange2', text) + + case ts.ClassificationType.numericLiteral: + return colorize('paleGreen', text) + + case ts.ClassificationType.identifier: + return colorize('paleCyan', text) + + + } + + return text +} + +function colorizeText(text: string, spans: ClassifiedSpan[]): string { + for (let i = spans.length - 1; i >= 0; i--) { + const s = spans[i] + const end = s.start + s.length + text = text.slice(0, s.start) + colorizeSpan(text.slice(s.start, end), s) + text.slice(end) + } + + return text +} + +function formatDiagnostic(diag: Diagnostic, opt?: FormatDiagnosticOptions) { + +} + +function maybeGetBetterNode(node: ts.Node) { + if ((ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node) || ts.isFunctionExpression(node) || ts.isClassExpression(node)) && node.name) { + return node.name + } + + if (ts.isVariableDeclaration(node)) { + return node.name + } + + return node +} + +async function renderSnippets(locations: NodeOrSource[], context = 2) { + if (locations.length === 0) { + return + } + + const mapped = locations.map(n => isSourceLocation(n) ? n : annotateNode(n, '')) + + for (let i = 0; i < mapped.length; i++) { + const location = mapped[i] + const previous = mapped[i-1] + + let ctx = context + if (previous?.fileName === location.fileName) { + if ((location.line - context) <= previous.line && location.line >= previous.line) { + ctx = location.line - previous.line - 1 + } + } else { + printLine(`${path.relative(getWorkingDir(), location.fileName)}:${location.line}:${location.column}`) + } + + const classifications = await getClassifications(location.fileName, { + start: { line: location.line - (ctx + 1), column: 0 }, + end: { line: location.line, column: 0 } + }) + + + const text = classifications.sourceFile.getFullText() + const snippet = getSourceSnippet(location, text, { context: ctx, colorizer: (text, line, column) => { + const pos = classifications.getOffset({ line, column }) + const spans = classifications.spans + .filter(s => s.start + s.length >= pos && s.start <= pos + text.length) + .map(s => ({ ...s, start: s.start - pos })) + + return colorizeText(text, spans) + }}) + + for (const l of snippet) { + printLine(l) + } + } +} + +export function annotateNode(node: ts.Node, message: string): SourceLocation { + return getNodeLocation(maybeGetBetterNode(ts.getOriginalNode(node)), message) +} + +type NodeOrSource = ts.Node | SourceLocation + +function isSourceLocation(obj: NodeOrSource): obj is SourceLocation { + return 'line' in obj && 'column' in obj && 'fileName' in obj +} + +export function compilerError(message: string, ...locations: NodeOrSource[]): never { + const fn = AsyncResource.bind(async () => { + await renderSnippets(locations) + printLine() + printLine(colorize('brightRed', `ERROR `) + colorize('brightWhite', `${message}`)) + }) + + throw new RenderableError(message, fn) +} diff --git a/src/compiler/entrypoints.ts b/src/compiler/entrypoints.ts new file mode 100644 index 0000000..a555825 --- /dev/null +++ b/src/compiler/entrypoints.ts @@ -0,0 +1,364 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { escapeRegExp, failOnNode, isRelativeSpecifier, keyedMemoize, memoize, throwIfNotFileNotFoundError } from '../utils' +import { ResolvedProgramConfig } from './config' +import { getFs } from '../execution' +import { getLogger } from '..' +import { getFileHasher } from './incremental' +import { getProgramFs } from '../artifacts' +import { JsonFs } from '../system' +import { getWorkingDir } from '../workspaces' + +// The "entrypoint" mechanic is primarily used to ease onboarding. +// +// Synapse doesn't require an entrypoint because distributed applications +// are, generally speaking, always running. There's nothing to start. +// +// Most programmers will not be comfortable with this concept. When they +// run a program, they expect some code will be executed and then, usually, stop. +// That's how it's been for 100 years or so. And, for many applications, that's +// how it will continue to be. +// +// So instead of trying to teach users a new way of thinking right away, we'll +// gradually introduce them to new ideas by first starting with familiar ones. +// +// For Synapse, declaring a `main` function is interpretted as "create an executable" +// We can do this without the user explicitly requesting an executable because +// it's a purely additive feature. The distributed application does not change. + +// A function is potentially a executable entrypoint if: +// * Named `main` +// * Exported +// * Is assignable to this signature: +// ```ts +// (...args: string[]) => Promise | number | void +// ``` +// +// Contrary to many other conventions, `main` will _not_ be called with +// the targeted executable as `args[0]`. And, in contrast to NodeJS, +// `args[1]` will _not_ contain the target script. +// +// In other words, `args` is equal to `process.argv.slice(2)`. +// +// Reasoning behind this decision: +// * The first two arguments in `process.argv` establish a context and +// generally aren't as relevant to applications as the remaining args +// +// * `process.argv` is still available if truly needed, and many alternatives are also available: +// * `argv[0]` has "resolved" substitutes e.g. `process.execPath` +// * `argv[1]` can be replaced with `__filename` or `import.meta.filename` in most cases +// +// Motivation for using the varargs signature: +// * Better expresses how the application will be called +// * Allows for establishing basic program expectations +// +// For example, a simple program that only takes two inputs can be written as: +// ```ts +// export function main(a: string, b: string) { ... } +// ``` +// +// Which means this program must _never_ be invoked with anything less than two parameters. +// + +type Main = (...args: string[]) => Promise | number | void + +// process.exitCode <-- sets the exit code on exit + +export function isMainFunction(node: ts.FunctionDeclaration, getTypeChecker?: () => ts.TypeChecker) { + if (node.name?.text !== 'main') { + return false + } + + const mods = ts.getModifiers(node) + if (!mods?.find(m => m.kind === ts.SyntaxKind.ExportKeyword)) { + return false + } + + if (!getTypeChecker) { + return true + } + + const typeChecker = getTypeChecker() + const sig = typeChecker.getSignatureFromDeclaration(node) + if (!sig) { + getLogger().debug(`Skipped node due to missing signature`) + return false + } + + const params = sig.getParameters() + for (const p of params) { + const decl = p.valueDeclaration + if (!decl) { + failOnNode(`Missing parameter value declaration: ${p.name}`, node) + } + + if (!ts.isParameter(decl)) { + failOnNode(`Not a parameter`, decl) + } + + const type = typeChecker.getTypeOfSymbol(p) + if (decl.dotDotDotToken) { + // TODO: support tuples? + // if (!typeChecker.isArrayType(type)) { + // getLogger().debug(`Skipped node due to missing signature`) + // // failOnNode('Not an array type', decl) + // return false + // } + + const elementType = type.getNumberIndexType() + if (!elementType) { + getLogger().debug(`Skipped node due to missing element type`) + + // failOnNode('Expected an element type', decl) + return false + } + + if (!typeChecker.isTypeAssignableTo(typeChecker.getStringType(), elementType.getNonNullableType())) { + // failOnNode('Not a string type', decl) + return false + } + } else { + if (!typeChecker.isTypeAssignableTo(typeChecker.getStringType(), type.getNonNullableType())) { + // failOnNode('Not a string type', decl) + return false + } + } + } + + return true +} + +export function hasMainFunction(sf: ts.SourceFile, getTypeChecker?: () => ts.TypeChecker) { + const statements = sf.statements + const len = statements.length + for (let i = 0; i < len; i++) { + // Duplicated `name` check to avoid function call + const s = statements[i] + if (s.kind !== ts.SyntaxKind.FunctionDeclaration || (s as ts.FunctionDeclaration).name?.text !== 'main') continue + + if (isMainFunction(s as ts.FunctionDeclaration, getTypeChecker)) { + return true + } + } + + return false +} + +// Module helpers + +const synapseScheme = 'synapse:' +const providerScheme = 'synapse-provider:' +export function findProviderImports(sourceFile: ts.SourceFile) { + const providers = new Set() + for (const statement of sourceFile.statements) { + if (ts.isImportDeclaration(statement)) { + const spec = (statement.moduleSpecifier as ts.StringLiteral).text + if (spec.startsWith(providerScheme)) { + providers.add(spec.slice(providerScheme.length)) + } + } + } + return providers +} + +function findInterestingSpecifiers(sf: ts.SourceFile, resolveBareSpecifier: (spec: string) => string | undefined) { + const zig = new Set() + const bare = new Set() + for (const s of sf.statements) { + if (!ts.isImportDeclaration(s) && !ts.isExportDeclaration(s)) continue + if (!s.moduleSpecifier) continue + + const spec = (s.moduleSpecifier as ts.StringLiteral).text + if (isRelativeSpecifier(spec)) { + if (s.kind === ts.SyntaxKind.ImportDeclaration && spec.endsWith('.zig')) { + zig.add(spec) + } + continue + } + + const r = resolveBareSpecifier(spec) + if (!r) { + bare.add(spec) + } + } + + return { bare, zig } +} + +function createSpecifierResolver(cmd: ts.ParsedCommandLine, dir: string) { + const baseUrl = cmd.options.baseUrl + const paths = cmd.options.paths + const resolveDir = baseUrl ?? dir + const files = new Set(cmd.fileNames) + const fs = getFs() + + function resolvePaths(paths: Record) { + const r: [string | RegExp, string[]][] = [] + + const getPrefixLength = keyedMemoize((s: string) => { + const idx = s.indexOf('*') + + return idx === -1 ? s.length : idx + }) + + const entries = Object.entries(paths) + .filter(([k, v]) => v.length > 0) + .sort((a, b) => getPrefixLength(b[0]) - getPrefixLength(a[0])) + + for (const [k, v] of entries) { + const index = getPrefixLength(k) + const resolved = v.map(x => path.resolve(resolveDir, x)) + if (index === k.length) { + r.push([k, resolved]) + continue + } + + const left = escapeRegExp(k.slice(0, index)) + const right = escapeRegExp(k.slice(index + 1)) + const pattern = new RegExp(`^${left}${'(.*)'}${right}$`) + r.push([pattern, resolved]) + } + + return r + } + + const getResolvedPaths = memoize(() => paths ? resolvePaths(paths) : []) + const perfile = new Set() + const mappings = new Map() + + function resolveBareSpecifier(spec: string) { + if (!baseUrl && !paths) { + return + } + + if (mappings.has(spec)) { + const m = mappings.get(spec) + if (m) { + perfile.add(spec) + } + return m + } + + if (baseUrl) { + const p = path.resolve(baseUrl, spec) + const withExt = path.extname(p) === '' ? `${p}.ts` : p // FIXME + if (files.has(withExt)) { + perfile.add(spec) + mappings.set(spec, withExt) + return withExt + } + } + + const patterns = getResolvedPaths() + for (const [pattern, locations] of patterns) { + if (typeof pattern === 'string') { + if (pattern !== spec) continue + for (const l of locations) { + const withExt = path.extname(l) === '' ? `${l}.ts` : l // FIXME + if (files.has(withExt) || fs.fileExistsSync(withExt)) { + perfile.add(spec) + mappings.set(spec, withExt) + return withExt + } + } + continue + } + + const m = spec.match(pattern) + if (!m) continue + + const wildcard = m[1] + for (const l of locations) { + const p = l.replace('*', wildcard) + const withExt = path.extname(p) === '' ? `${p}.ts` : p // FIXME + if (files.has(withExt) || fs.fileExistsSync(withExt)) { + perfile.add(spec) + mappings.set(spec, withExt) + return withExt + } + } + } + + mappings.set(spec, undefined) + } + + function getPerFileMappings() { + if (perfile.size === 0) { + return + } + + const keys = [...perfile] + perfile.clear() + + return Object.fromEntries(keys.map(k => [k, mappings.get(k)!])) + } + + return { mappings, resolveBareSpecifier, getPerFileMappings } +} + + + +export async function findAllBareSpecifiers(config: ResolvedProgramConfig, host: ts.CompilerHost) { + const bare = new Set() + const workingDir = getWorkingDir() + const resolver = createSpecifierResolver(config.tsc.cmd, workingDir) + const target = config.tsc.cmd.options.target ?? ts.ScriptTarget.Latest + const hasher = getFileHasher() + const incremental = config.csc.incremental + const [hashes, previousSpecs] = await Promise.all([ + Promise.all(config.tsc.cmd.fileNames.map(async f => [f, await hasher.getHash(f)])), + !incremental ? undefined : getSpecData(), + ]) + + const specs = previousSpecs ?? {} + for (const [f, h] of hashes) { + const relPath = path.relative(workingDir, f) + if (specs[relPath]?.hash === h) { + specs[relPath].specs.forEach(s => bare.add(s)) + continue + } + + // We're probably double parsing files. Not sure if the + // standalone compiler host does any caching. + const sf = host.getSourceFile(f, target, err => getLogger().error(err)) + if (!sf) { + continue + } + + const results = findInterestingSpecifiers(sf, resolver.resolveBareSpecifier) + const hasZigImport = results.zig.size > 0 ? true : undefined + specs[relPath] = { hash: h, specs: [...results.bare], mappings: resolver.getPerFileMappings(), hasZigImport } + for (const d of results.bare) { + bare.add(d) + } + } + + await saveSpecData(specs) + + return bare +} + +const specsFileName = `[#sources]__bareSpecifiers__.json` + +// `mappings` and `specs` will likely have many duplicate elements across all files +// need to add a fast way to encode/decode +interface SpecDataElement { + readonly hash: string + readonly specs: string[] + readonly mappings?: Record + readonly hasZigImport?: boolean +} + +async function saveSpecData(data: Record, fs: Pick = getProgramFs()) { + await fs.writeJson(specsFileName, data) +} + +async function getSpecData(fs: Pick = getProgramFs()) { + return await fs.readJson>(specsFileName).catch(throwIfNotFileNotFoundError) +} + +const _getSpecData = memoize(() => getSpecData()) + +export async function hasZigImport(relPath: string) { + return !!(await _getSpecData())?.[relPath].hasZigImport +} \ No newline at end of file diff --git a/src/compiler/esm.ts b/src/compiler/esm.ts new file mode 100644 index 0000000..dc0716f --- /dev/null +++ b/src/compiler/esm.ts @@ -0,0 +1,171 @@ +import ts from 'typescript' +import { createVariableStatement } from '../utils' + +// 1. normalize all import declarations to the namespace form e.g. `import * as foo from 'foo'` +// 2. change the namespace identifier e.g. `__foo_namespace` instead of `foo` +// 3. emit a wrapped assignment `const foo = __wrapExports('foo', import.meta.__importer, __foo_namespace) +// 4. use wrapped namespace to create original bindings, e.g. for default use `__wrapExports('foo', import.meta.__importer, __foo_namespace).default` + + +// TODO: wrap dynamic imports too + +export function transformImports(sf: ts.SourceFile) { + const imports: ts.Statement[] = [] + const wraps: ts.Statement[] = [] + const bindings: ts.Statement[] = [] + + const didEmit = new Set() + const specMap = new Map() + function specToIdent(spec: string) { + if (specMap.has(spec)) { + return specMap.get(spec)! + } + + const val = `__import_${specMap.size}` + specMap.set(spec, val) + + return val + } + + function createWrap(spec: ts.Expression, namespace: ts.Expression) { + const importer = ts.factory.createPropertyAccessExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('import'), + 'meta' + ), + '__virtualId' + ) + + return ts.factory.createCallExpression( + ts.factory.createIdentifier('__wrapExports'), + undefined, + [spec, importer, namespace] + ) + } + + function wrapNamespace(spec: string, attributes?: ts.ImportAttributes) { + const name = specToIdent(spec) + if (didEmit.has(spec)) { + return name + } + + didEmit.add(spec) + + const namespace = name + '_namespace' + + const importDecl = ts.factory.createImportDeclaration( + undefined, + ts.factory.createImportClause(false, undefined, ts.factory.createNamespaceImport(ts.factory.createIdentifier(namespace))), + ts.factory.createStringLiteral(spec), + attributes + ) + + imports.push(importDecl) + + const wrapped = createWrap(ts.factory.createStringLiteral(spec), ts.factory.createIdentifier(namespace)) + wraps.push(createVariableStatement(name, wrapped)) + + return name + } + + function createAlias(spec: string, name: string, propertyName = name, attributes?: ts.ImportAttributes) { + const wrappedName = wrapNamespace(spec, attributes) + + return createVariableStatement( + name, + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier(wrappedName), + propertyName, + ) + ) + } + + function transform(decl: ts.ImportDeclaration) { + const clause = decl.importClause + if (!clause) { // `import 'foo'` + return decl + } + + if (clause.isTypeOnly) { + return decl + } + + const spec = (decl.moduleSpecifier as ts.StringLiteral).text + + const currentBindings: ts.Statement[] = [] + if (clause.name) { // import foo from 'foo' + currentBindings.push( + createAlias(spec, clause.name.text, 'default', decl.attributes) + ) + } + + const namedBindings = clause.namedBindings + if (namedBindings) { + if (ts.isNamedImports(namedBindings)) { + for (const binding of namedBindings.elements) { + if (binding.isTypeOnly) continue + + currentBindings.push( + createAlias(spec, binding.name.text, binding.propertyName?.text, decl.attributes) + ) + } + } else { + currentBindings.push( + createVariableStatement( + namedBindings.name, + ts.factory.createIdentifier(wrapNamespace(spec, decl.attributes)), + ) + ) + } + } + + if (currentBindings.length === 0) { + return decl + } + + bindings.push(...currentBindings) + + return ts.factory.createNotEmittedStatement(decl) + } + + function wrapDynamicImport(node: ts.CallExpression) { + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression(node, 'then'), + undefined, + [ + ts.factory.createArrowFunction( + undefined, + undefined, + [ts.factory.createParameterDeclaration(undefined, undefined, 'namespace')], + undefined, + undefined, + // TODO: this evaluates the specifier expression twice (which can cause problems!!!) + createWrap(node.arguments[0], ts.factory.createIdentifier('namespace')) + ) + ] + ) + } + + const mapped = sf.statements.map(s => { + if (!ts.isImportDeclaration(s)) { + return s + } + + return transform(s) + }) + + if (imports.length === 0) { + return sf + } + + return ts.factory.updateSourceFile( + sf, + [ + ...imports, + ...wraps, + ...bindings, + ...mapped, + ] + ) +} + diff --git a/src/compiler/host.ts b/src/compiler/host.ts new file mode 100644 index 0000000..9d34649 --- /dev/null +++ b/src/compiler/host.ts @@ -0,0 +1,793 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { Mutable, makeRelative, memoize, resolveRelative, sortRecord, strcmp, tryReadJson } from '../utils' +import { createTranspiler } from '../bundler' +import { createrLoader } from '../loader' +import { ResourceTransformer, generateModuleStub, getFqnComponents } from './transformer' +import { createPermissionsBinder } from '../permissions' +import { runTask } from '../logging' +import { CompiledFile, createGraphCompiler, createRuntimeTransformer, createSerializer, getModuleType } from '../static-solver' +import { getOrCreateDeployment, getWorkingDir } from '../workspaces' +import { Fs, JsonFs, createLocalFs } from '../system' +import { PackageService, createPackageService, resolveBareSpecifier, resolveDeferredTargets } from '../pm/packages' +import { SourceMapHost, emitChunk } from '../static-solver/utils' +import { getLogger } from '../logging' +import { createSourceMapParser, getArtifactSourceMap, hydratePointers } from '../runtime/loader' +import { Artifact, CompiledChunk, LocalMetadata, ModuleBindingResult, createArtifactFs, getArtifactFs, getDataRepository, getProgramFs, getTargets, readInfraMappings, readModuleManifest, setTargets, toFs, writeInfraMappings, writeModuleManifest, } from '../artifacts' +import { CombinedOptions } from '..' +import { ModuleResolver, createModuleResolver } from '../runtime/resolver' +import { SourceMapV3 } from '../runtime/sourceMaps' +import { TemplateService } from '../templates' +import { DeclarationFileHost, createDeclarationFileHost } from './declarations' +import { ResourceTypeChecker } from './resourceGraph' +import { coerceToPointer, isDataPointer, pointerPrefix, toAbsolute } from '../build-fs/pointers' +import { getBuildTargetOrThrow, getFs } from '../execution' +import { resolveValue } from '../runtime/modules/serdes' +import { getCurrentPkg, getPackageJson } from '../pm/packageJson' +import { getOutputFilename } from './config' +import { getFileHasher } from './incremental' +import { transformImports } from './esm' +import { createBasicDataRepo } from '../runtime/rootLoader' + + +interface SynthOptions { + readonly deployTarget?: string + // readonly backend?: Backend + readonly entrypoint?: string + + // For modules + readonly generateExports?: boolean +} + +export interface CompilerOptions extends SynthOptions { + readonly noInfra?: boolean + readonly noSynth?: boolean + readonly sharedLib?: boolean + readonly workingDirectory?: string + readonly incremental?: boolean + readonly includeJs?: boolean + readonly targetFiles?: string | string[] + readonly excludeProviderTypes?: boolean + readonly stripInternal?: boolean // TSC option + + // For compiling native code + readonly hostTarget?: string + + readonly environmentName?: string +} + +export interface ReplacementSymbol { + readonly moduleSpecifier: string // Always relative to the pkg dir + readonly symbolName: string +} + +export interface TargetsFile { + [fileName: string]: Record +} + +function fixSourcemapSources(sourcemap: string, outFile: string, sourceFile: string) { + return JSON.stringify({ + ...JSON.parse(sourcemap), + sources: [path.relative(path.dirname(outFile), sourceFile)] + }) +} + +interface RenderedFile { + name: string + runtime: { text: string; sourcemap?: Uint8Array } + infra: { text: string; sourcemap?: Uint8Array } + sourceDelta?: { line: number; column: number } +} + +export class CompilerHost { + private readonly rootDir: string + private readonly infraFiles = new Map>() + private readonly runtimeFiles = new Map>() + private readonly declarationHost: DeclarationFileHost + + public constructor( + public readonly sourceMapHost: SourceMapHost, + private readonly graphCompiler: ReturnType, + private readonly resourceTransformer: ResourceTransformer, + private readonly resourceTypeChecker: ResourceTypeChecker, + private readonly compilerOptions: ts.CompilerOptions, + public readonly opt: CompilerOptions = {}, + ) { + this.rootDir = this.compilerOptions.rootDir ?? this.opt.workingDirectory! + this.program = this.createProgram() + this.declarationHost = createDeclarationFileHost(getProgramFs(), sourceMapHost) + } + + public getOutputFilename(fileName: string) { + return getOutputFilename(this.rootDir, this.compilerOptions, fileName) + } + + private async saveTypes(incremental = false) { + const programFs = getProgramFs() + + const bt = getBuildTargetOrThrow() + const getDest = (p: string) => makeRelative(bt.workingDirectory, this.getOutputFilename(`${p}.ts`)) + const targets: TargetsFile = {} + + async function getPackageEntrypoint() { + const pkgJson = await getCurrentPkg() + if (!pkgJson) { + throw new Error(`No package found. Target bindings can only be added from a package.`) + } + + const resolved = resolveBareSpecifier(pkgJson.data.name, pkgJson.data, 'cjs') + + return path.resolve(pkgJson.directory, resolved.fileName) + } + + if (this.program.resourceTransformer.bindings.size === 0) { + return + } + + const packageEntrypoint = await getPackageEntrypoint() + + for (const [k, v] of this.program.resourceTransformer.bindings.entries()) { + const { name, module } = getFqnComponents(k) + + const bindings = v.map(x => { + const bindingFqn = getFqnComponents(x.replacement) + const relativeModule = makeRelative( + path.dirname(packageEntrypoint), + path.resolve(bt.workingDirectory, getDest(bindingFqn.module)) + ) + + return [x.target, { moduleSpecifier: `./${relativeModule}`, symbolName: bindingFqn.name }] as const + }) + + const isExternalImport = !module.startsWith(bt.workingDirectory) + const key = isExternalImport ? module : `./${getDest(module)}` + targets[key] = { + ...targets[key], + [name]: Object.fromEntries(bindings), + } + } + + const existingTargetsFile = incremental ? await getTargets(programFs) : undefined + if (!existingTargetsFile) { + await setTargets(programFs, targets) + + return + } + + for (const [k, v] of Object.entries(targets)) { + const existingTargets = existingTargetsFile[k] + if (!existingTargets) { + existingTargetsFile[k] = v + continue + } + for (const [name, bindings] of Object.entries(v)) { + existingTargets[name] = { + ... existingTargets[name], + ...bindings, + } + } + } + + await setTargets(programFs, existingTargetsFile) + } + + private async renderArtifact(path: string, data: string, outfile: string, infra = false, oldSourcemap?: SourceMapV3) { + // Make sure 'infra' compiles don't collide w/ runtime builds + if (infra) { + path = path.replace(/\.([tj]sx?)$/, `.infra.$1`) + outfile = outfile.replace(/\.([tj]sx?)$/, `.infra.$1`) + } + + const transpiler = await this.getTranspiler() + const res = await transpiler.transpile( + path, + data, + outfile, + { + oldSourcemap, + sourcemapType: oldSourcemap ? 'external' : undefined, + } + ) + + return { + text: Buffer.from(res.result.contents).toString('base64'), + //textHash: result.hash, + sourcemap: res.sourcemap?.contents, + //sourcemapHash: sourcemap?.hash, + } + } + + private async emitArtifacts(fileName: string, moduleId?: string) { + const programFs = getProgramFs() + + const compiled = this.renderedFiles.get(fileName) + if (!compiled) { + // These files have no function/class declarations + getLogger().warn(`Missing artifacts for file: ${fileName}`) + + return {} + } + + const bt = getBuildTargetOrThrow() + const source = path.relative(bt.workingDirectory, fileName) + const key = `[#compile/${source}]` + + const rendered = await Promise.all(compiled) + + const artifacts = await Promise.all(rendered.map(async v => { + const [runtime, infra] = await Promise.all([ + v.runtime.sourcemap ? programFs.writeData(key, v.runtime.sourcemap) : undefined, + v.infra.sourcemap ? programFs.writeData(key, v.infra.sourcemap) : undefined, + ]) + + const sourcemaps = runtime && infra ? { runtime, infra } : undefined + + const result: CompiledChunk = { + kind: 'compiled-chunk', + infra: v.infra.text, + runtime: v.runtime.text, + } + + const pointer = await programFs.writeData( + key, + Buffer.from(JSON.stringify(result), 'utf-8'), + { metadata: { sourcemaps, moduleId, sourceDelta: v.sourceDelta } }, // TODO: add support for using pointer as metadata + ) + + return { + source, + name: v.name, + data: result, + pointer, + } + })) + + return Object.fromEntries(artifacts.sort((a, b) => strcmp(a.name, b.name)).map(a => [a.name, a] as const)) + } + + private readonly moduleManifest: Record = {} + private readonly infraFileMapping: Record = {} + private readonly sources: Sources = {} + private readonly pointers: Record> = {} + + private async writeManifest(incremental?: boolean) { + const programFs = getProgramFs() + const oldManifest = incremental ? await readModuleManifest(programFs) : undefined + await Promise.all(this.sourcePromises.values()) + + // XXX: we do this to support incremental builds + // The declaration emitter needs to know about other bindings to rewrite imports/exports + for (const [k, v] of Object.entries({ ...oldManifest, ...this.moduleManifest })) { + const outfile = this.getOutputFilename( + path.resolve(getWorkingDir(), k) + ).replace(/\.js$/, '.d.ts') + + this.declarationHost.setBinding(outfile, v.id) + } + + for (const [k, v] of Object.entries(this.moduleManifest)) { + const r = this.declarationHost.transformModuleBinding(k, v.id) + const [text, sourcemap] = await Promise.all([ + programFs.writeData(`[#compile/${k}]`, Buffer.from(r.text, 'utf-8')), + programFs.writeData(`[#compile/${k}]`, Buffer.from(r.sourcemap, 'utf-8')), + ]) + + this.moduleManifest[k] = { + ...this.moduleManifest[k], + types: { + name: r.name, + text, + sourcemap, + references: r.references, + symbols: this.resourceTypeChecker.getFileSymbols( + resolveRelative(getWorkingDir(), k) + ), + }, + } + } + + const updateManifest = async () => { + if (incremental && Object.keys(this.moduleManifest).length === 0) { + return + } + + const updated = sortRecord({ ...oldManifest, ...this.moduleManifest }) + await writeModuleManifest(programFs, updated) + } + + const updateMappings = async () => { + if (incremental && Object.keys(this.infraFileMapping).length === 0) { + return + } + + const oldMappings = incremental ? await readInfraMappings(programFs) : undefined + await writeInfraMappings(programFs, sortRecord({ ...oldMappings, ...this.infraFileMapping })) + } + + const updateSources = async () => { + if (incremental && Object.keys(this.sources).length === 0) { + return + } + + const oldSources = incremental ? await readSources(programFs) : undefined + await writeSources(programFs, sortRecord({ ...oldSources, ...this.sources })) + } + + const updatePointers = async () => { + if (incremental && Object.keys(this.pointers).length === 0) { + return + } + + const oldPointers = incremental ? await readPointersFile(programFs) : undefined + await writePointersFile(programFs, sortRecord({ ...oldPointers, ...this.pointers })) + } + + await Promise.all([ + updateManifest(), + updateMappings(), + updateSources(), + updatePointers(), + ]) + } + + public async finish(incremental?: boolean) { + await Promise.all([ + this.saveTypes(incremental), + ...(this.runtimeFiles.values()), + ...(this.infraFiles.values()), + ]) + + return this.writeManifest(incremental) + } + + private readonly program: Program + + private readonly renderedFiles = new Map[]>() + private createProgram(): Program { + const resourceTransformer = this.resourceTransformer + const outDir = this.compilerOptions.outDir ?? this.rootDir + const render = async (f: CompiledFile) => { + const relPath = this.getOutputFilename(f.source) + const virtualOutfile = relPath.replace(/\.(?:t|j)(sx?)$/, `-${f.artifactName.slice(0, 48)}.js`) + const inputPath = path.resolve(outDir, path.relative(this.rootDir, f.path)) // Looks weird. Need to do this because TypeScript source maps assume it was emitted in the outdir + + const [runtime, infra] = await Promise.all([ + this.renderArtifact(inputPath, f.data, virtualOutfile, false, f.sourcesmaps?.runtime), + this.renderArtifact(inputPath, f.infraData, virtualOutfile, true, f.sourcesmaps?.infra), + ]) + + return { name: f.artifactName, runtime, infra, sourceDelta: resourceTransformer.getDeltas(f.sourceNode) } + } + + this.graphCompiler.onEmitFile(f => { + const arr = this.renderedFiles.get(f.source) ?? [] + arr.push(render(f)) + this.renderedFiles.set(f.source, arr) + }) + + return { + graphCompiler: this.graphCompiler, + resourceTransformer, + } + } + + private readonly sourcePromises = new Map>() + public addSource(name: string, outfile: string, isTsArtifact?: boolean) { + const p = getFileHasher().getHash(path.resolve(this.rootDir, name)).then(hash => { + this.sources[name] = { outfile, hash, isTsArtifact } + }) + this.sourcePromises.set(name, p) + } + + public async saveTscFile(sourceFile: ts.SourceFile, fileName: string, text: string, moduleId?: string, internal?: boolean) { + const bt = getBuildTargetOrThrow() + const programFs = getProgramFs() + const source = makeRelative(bt.workingDirectory, sourceFile.fileName) + const dest = makeRelative(bt.workingDirectory, fileName) + + if (moduleId) { + this.addSource(source, dest) + this.moduleManifest[source] = { id: moduleId, path: dest, internal } + } + + if (fileName.endsWith('.d.ts')) { + this.declarationHost.addDeclaration(source, dest, text) + } else if (fileName.endsWith('.d.ts.map')) { + this.declarationHost.addSourcemap(dest.replace(/\.map$/, ''), text) + } + + await programFs.writeFile(`[#compile/${source}]${dest}`, text) + } + + private readonly getTranspiler = memoize(async () => { + return createTranspiler( + getFs(), + undefined, + this.compilerOptions, + ) + }) + + public async emitNoInfra(sourceFile: ts.SourceFile, needsDeploy?: boolean) { + if (needsDeploy) { + sourceFile = generateModuleStub(this.rootDir, this.graphCompiler, ts.getOriginalNode(sourceFile) as ts.SourceFile) + } + + // const handlers = !needsDeploy ? this.resourceTransformer.getReflectionTransformer() : undefined + const { text } = emitChunk(this.sourceMapHost, sourceFile, undefined, { emitSourceMap: false, removeComments: true }) + const outfile = this.getOutputFilename(sourceFile.fileName) + const transpiler = await this.getTranspiler() + const res = await transpiler.transpile( + sourceFile.fileName, + text, + outfile, + ) + + const fs = getFs() + await fs.writeFile(outfile, res.result.contents) + } + + private async emitSourceFile(sourceFile: ts.SourceFile, isInfra?: boolean, metadata?: LocalMetadata) { + const bt = getBuildTargetOrThrow() + const outfile = this.getOutputFilename(sourceFile.fileName) + const resolvedOutfile = isInfra ? outfile.replace(/\.js$/, '.infra.js') : outfile + const relSourcefile = makeRelative(bt.workingDirectory, sourceFile.fileName) + const relOutfile = makeRelative(bt.workingDirectory, resolvedOutfile) + + const emitSourceMap = !!this.compilerOptions.sourceMap + // XXX: we are always stripping comments here because it's possible for the generated code to render + // malformed if there are any isolated comments at the end of the source file + const { text, sourcemap } = emitChunk(this.sourceMapHost, sourceFile, undefined, { emitSourceMap, removeComments: true }) + + const transpiler = await this.getTranspiler() + const res = await transpiler.transpile( + sourceFile.fileName, + text, + resolvedOutfile, + { + sourcemapType: emitSourceMap ? 'linked' : undefined, + oldSourcemap: sourcemap, + } + ) + + const compiledText = res.result.text + const fixedSourcemap = res.sourcemap !== undefined + ? fixSourcemapSources(res.sourcemap.text, resolvedOutfile, sourceFile.fileName) + : undefined + + const programFs = getProgramFs() + + if (fixedSourcemap) { + await programFs.writeFile(`[#compile/${relSourcefile}]${relOutfile}.map`, fixedSourcemap) + } + + if (resolvedOutfile !== outfile) { + this.infraFileMapping[makeRelative(bt.workingDirectory, outfile)] = relOutfile + } + + this.addSource(relSourcefile, makeRelative(bt.workingDirectory, outfile)), + await programFs.writeFile(`[#compile/${relSourcefile}]${relOutfile}`, compiledText, { metadata }) + + return { compiledText, relOutfile } + } + + public compileSourceFileRuntimeOnly(sourceFile: ts.SourceFile) { + const runtimeTransformer = createRuntimeTransformer(this.program.graphCompiler, this.resourceTypeChecker) + sourceFile = runtimeTransformer(sourceFile) as ts.SourceFile + const p = this.emitSourceFile(sourceFile).then(r => r.compiledText) + this.runtimeFiles.set(sourceFile.fileName, p) + + return p + } + + public async emitDeployStub(sourceFile: ts.SourceFile) { + const bt = getBuildTargetOrThrow() + const relSourcefile = makeRelative(bt.workingDirectory, sourceFile.fileName) + const emitSourceMap = !!this.compilerOptions.sourceMap + const transpiler = await this.getTranspiler() + const programFs = getProgramFs() + const outfile = this.getOutputFilename(sourceFile.fileName) + + const stub = generateModuleStub(this.rootDir, this.graphCompiler, ts.getOriginalNode(sourceFile) as ts.SourceFile) + const { text, sourcemap } = emitChunk(this.sourceMapHost, stub, undefined, { emitSourceMap, removeComments: true }) + const res = await transpiler.transpile( + sourceFile.fileName, + text, + outfile, + { + sourcemapType: emitSourceMap ? 'linked' : undefined, + oldSourcemap: sourcemap, + } + ) + + const relOutfile = makeRelative(bt.workingDirectory, outfile) + const fixedSourcemap = res.sourcemap !== undefined + ? fixSourcemapSources(res.sourcemap.text, outfile, sourceFile.fileName) + : undefined + + await Promise.all([ + fixedSourcemap ? programFs.writeFile(`[#compile/${relSourcefile}]${relOutfile}.map`, fixedSourcemap) : undefined, + programFs.writeFile(`[#compile/${relSourcefile}]${relOutfile}`, res.result.text) + ]) + } + + public compileSourceFile(sourceFile: ts.SourceFile, writeToDisk?: boolean, moduleId?: string, internal?: boolean) { + // Defensive check + const previous = this.infraFiles.get(sourceFile.fileName) + if (previous) { + return previous + } + + const serializerTransformer = createSerializer(this.program.graphCompiler, this.resourceTypeChecker).createTransformer( + undefined, + node => this.program.resourceTransformer.visitAsInfraChunk(node), + ) + + sourceFile = serializerTransformer.visit(sourceFile) as ts.SourceFile + sourceFile = this.program.resourceTransformer.visitSourceFile(sourceFile) // Possibly do this pass in `serializerTransformer` + + const importDecl = ts.factory.createImportDeclaration( + undefined, + ts.factory.createImportClause( + false, + undefined, + ts.factory.createNamedImports([ + ts.factory.createImportSpecifier( + false, + ts.factory.createIdentifier('__scope__'), + ts.factory.createIdentifier('scope') + ) + ]) + ), + ts.factory.createStringLiteral('synapse:core') + ) + + const emit = async () => { + const bt = getBuildTargetOrThrow() + const relSourcefile = makeRelative(bt.workingDirectory, sourceFile.fileName) + const artifacts = await this.emitArtifacts(sourceFile.fileName, moduleId) + + sourceFile = ts.factory.updateSourceFile( + sourceFile, + [ + importDecl, + ...sourceFile.statements, + ] + ) + + if (getModuleType(this.compilerOptions?.module) === 'esm') { + sourceFile = transformImports(sourceFile) + } + + const { compiledText, relOutfile } = await this.emitSourceFile(sourceFile, !writeToDisk, { + dependencies: Object.values(artifacts).map(v => v.pointer), + }) + + const pointers = Object.entries(artifacts).map(([k, v]) => [k, v.pointer] as const) + this.pointers[relOutfile] = Object.fromEntries(pointers) + + if (moduleId) { + this.moduleManifest[relSourcefile] = { id: moduleId, path: relOutfile, internal } + } + + return compiledText + } + + const p = emit() + this.infraFiles.set(sourceFile.fileName, p) + + return p + } +} + +type CompilationMode = + | 'passthrough' + | 'runtime' + | 'infra' + | 'infra-stub' + | 'no-synth' + +interface Sources { + [fileName: string]: { + hash: string + outfile: string + isTsArtifact?: boolean + } +} + +const sourcesFileName = `[#compile]__sources__.json` +async function writeSources(fs: Pick, sources: Sources) { + await fs.writeFile(sourcesFileName, JSON.stringify(sources)) +} + +export async function readSources(fs: Pick = getProgramFs()): Promise { + const s = await tryReadJson | Sources>(fs, sourcesFileName.slice('[#compile]'.length)) + + return s as Sources | undefined +} + +const pointersFileName = '[#compile]__pointers__.json' +async function writePointersFile(fs: Pick, pointers: Record>): Promise { + await fs.writeJson(pointersFileName, pointers) +} + +export async function readPointersFile(fs: Pick): Promise> | undefined> { + try { + return await fs.readJson(pointersFileName) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } +} + +export interface CompiledSource { + readonly name: string + // readonly text: string + readonly source: string +} + +interface SynthOptions { + readonly esm?: boolean // temporary + readonly outDir?: string + readonly sources?: CompiledSource[] + readonly compilerOptions?: CompilerOptions +} + +export async function synth(entrypoints: string[], deployables: string[], opt: SynthOptions = {}) { + const { + sources = [], + } = opt + + const afs = await getArtifactFs() + const deploymentId = await getOrCreateDeployment() + const bt = getBuildTargetOrThrow() + ;(bt as Mutable).deploymentId = deploymentId + process.env.SYNAPSE_ENV = bt.environmentName // XXX: not clean + + const store = await afs.getCurrentProgramStore().getSynthStore() + const vfs = toFs(bt.workingDirectory, store.afs, getFs()) + const repo = getDataRepository(getFs()) + const dataRepo = createBasicDataRepo(repo) + + const moduleResolver = createModuleResolver(vfs, bt.workingDirectory) + const { deferredTargets, infraFiles, resolver, pointers } = await runTask('init', 'resolver', async () => { + const pkgService = await createPackageService(moduleResolver) + const { stores, deferredTargets, infraFiles, pkgResolver, pointers } = await pkgService.loadIndex() + store.setDeps(stores) + + return { stores, deferredTargets, infraFiles, resolver: pkgResolver, pointers } + }, 1) + + // We do this after loading the index to account for any compiled `package.json` + const targets = resolveDeferredTargets(moduleResolver, deferredTargets) + const sourcemapParser = createSourceMapParser(vfs, moduleResolver, bt.workingDirectory) + + const getSource = (fileName: string, spec: string, virtualLocation: string, type: 'runtime' | 'infra' = 'runtime') => { + if (!spec.startsWith(pointerPrefix)) { + if (type === 'infra' && infraFiles[fileName]) { + // getLogger().log(`Mapped ${fileName} -> ${infraFiles[fileName]}`) + return vfs.readFileSync(infraFiles[fileName], 'utf-8') + } + return vfs.readFileSync(fileName, 'utf-8') + } + + const pointer = coerceToPointer(!isDataPointer(spec) && fileName.startsWith(pointerPrefix) ? fileName : spec) + const name = toAbsolute(pointer) + + sourcemapParser?.registerDeferredMapping(name, () => { + const { hash, storeHash } = pointer.resolve() + + return getArtifactSourceMap(dataRepo, hash, storeHash) + }) + + const data = hydratePointers(dataRepo, pointer) + const artifact = typeof data === 'string' ? JSON.parse(data) as Artifact : data as Artifact + switch (artifact.kind) { + case 'compiled-chunk': + const contents = type === 'infra' ? artifact.infra : artifact.runtime + + return Buffer.from(contents, 'base64').toString('utf-8') + case 'deployed': + if (artifact.rendered) { + return Buffer.from(artifact.rendered, 'base64').toString('utf-8') + } + + if (type === 'runtime') { + throw new Error(`Not implemented: ${fileName}`) + } + + return resolveValue( + artifact.captured, + { loadModule: (spec, importer) => runtime.createRequire2(importer ?? virtualLocation)(spec) }, + artifact.table, + runtime.globals + ) + default: + throw new Error(`Unknown object kind: ${(artifact as any).kind}`) + } + } + + const getSourceFile = (fileName: string) => { + const sf = ts.createSourceFile( + fileName, + getSource(fileName, fileName, fileName), // ??? + ts.ScriptTarget.ES2020, + true + ) + + return sf + } + + const perms = createPermissionsBinder() + const solver = perms.createCapturedSolver(getSourceFile) + let permsCount = 0 + // FIXME: `thisArg` is a hack used for the specific case of checking ctor perms + // would be cleaner to have a different function handle this case + const solvePerms = (target: any, getContext: (t: any) => any, globals?: { console?: any }, args?: any[], thisArg?: any) => { + return runTask('perms', target.name ?? `fn-${permsCount++}`, () => { + return solver.evaluate(target, getContext, globals, args, thisArg) + }, 10) + } + + const loader = createrLoader( + store.afs, + targets, + infraFiles, + Object.fromEntries(deployables.map(x => [x, true])), + pointers, + resolver, + moduleResolver, + bt, + deploymentId, + sourcemapParser, + { + ...opt.compilerOptions, + backend: {}, + outDir: opt.outDir, + workingDirectory: bt.workingDirectory, + } + ) + + const getInfraSource = (fileName: string, id: string, virtualLocation: string) => getSource(fileName, id, virtualLocation, 'infra') + const runtime = loader.createRuntime(sources, getInfraSource, solvePerms) + + // The target module is always the _source_ file + const workingDirectory = opt.compilerOptions?.workingDirectory ?? process.cwd() + const targetModules = entrypoints.map(x => path.resolve(workingDirectory, x)).map(x => { + const resolved = sources.find(s => s.source === x)?.name + if (!resolved) { + throw new Error(`Missing output file for source: ${x}`) + } + + return resolved + }) + + if (opt.esm) { + const { terraform, permissions } = await loader.synthEsm(targetModules, runtime) + + getLogger().emitCompileEvent({ + entrypoint: '', + template: terraform.main, + }) + + return terraform.main + } + + const { terraform, permissions } = loader.synth(targetModules, runtime) + + // if (permissions.length > 0) { + // getLogger().log(`Required permissions:`, permissions) + // } + + getLogger().emitCompileEvent({ + entrypoint: '', + template: terraform.main, + }) + + return terraform.main +} + +interface Program { + graphCompiler: ReturnType + resourceTransformer: ResourceTransformer +} diff --git a/src/compiler/incremental.ts b/src/compiler/incremental.ts new file mode 100644 index 0000000..0d3956b --- /dev/null +++ b/src/compiler/incremental.ts @@ -0,0 +1,403 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { createFileHasher, keyedMemoize, memoize, throwIfNotFileNotFoundError } from '../utils' +import { Fs } from '../system' +import { getGlobalCacheDirectory } from '../workspaces' +import { getFs } from '../execution' +import { getProgramFs } from '../artifacts' + +interface DependencyEdge { + readonly kind: 'import' | 'export' + readonly specifier: string + readonly isTypeOnly: boolean + readonly resolved?: ts.ResolvedModuleFull + + // For debugging + // readonly symbols?: any[] +} + +function isTypeOnly(decl: ts.ImportDeclaration | ts.ExportDeclaration) { + if (ts.isImportDeclaration(decl)) { + const clause = decl.importClause + if (!clause) { + return false + } + + if (clause.isTypeOnly) { + return true + } + + if (clause.name) { + return false + } + + const bindings = decl.importClause.namedBindings + if (bindings && ts.isNamedImports(bindings)) { + return bindings.elements.every(item => item.isTypeOnly) + } + + return false + } + + if (decl.isTypeOnly) { + return true + } + + if (decl.exportClause) { + if (ts.isNamedExports(decl.exportClause)) { + return decl.exportClause.elements.every(item => item.isTypeOnly) + } + } + + return false +} + +function getDependencies(sourceFile: ts.SourceFile, opt: ts.CompilerOptions, host: ts.ModuleResolutionHost = ts.sys) { + const edges: DependencyEdge[] = [] + for (const statement of sourceFile.statements) { + if (ts.isImportDeclaration(statement) || ts.isExportDeclaration(statement)) { + const spec = (statement.moduleSpecifier as ts.StringLiteral)?.text + if (!spec) continue + + const { resolvedModule } = ts.resolveModuleName(spec, sourceFile.fileName, opt, host) + edges.push({ + kind: ts.isImportDeclaration(statement) ? 'import' : 'export', + resolved: resolvedModule, + specifier: spec, + isTypeOnly: isTypeOnly(statement), + }) + } + } + + return edges +} + +export type CompilationGraph = Awaited> + +function getCompilationGraph( + rootFileNames: readonly string[], + getDeps: (fileName: string) => Dependency[] | void // Promise | Dependency[] | void, +) { + const visited = new Set() + const edges: [from: string, to: string][] = [] + const typeEdges = new Set() + const specifiers = new Map() + + function dfs(fileName: string) { + if (visited.has(fileName)) { + return + } + + visited.add(fileName) + + const deps = getDeps(fileName) + if (!deps) { + return + } + + const nextNodes = new Set() + function addEdge(from: string, to: string, specifier: string, isTypeOnly?: boolean) { + edges.push([from, to]) + nextNodes.add(to) + specifiers.set(edges.length - 1, specifier) + if (isTypeOnly) { + typeEdges.add(edges.length - 1) + } + } + + for (const d of deps) { + addEdge(fileName, d.fileName, d.specifier, d.isTypeOnly) + } + + [...nextNodes].forEach(dfs) + } + + rootFileNames.forEach(dfs) + + return { + files: [...visited.values()], + edges, + typeEdges, + specifiers, + } +} + +interface Dependency { + readonly fileName: string + readonly specifier: string + readonly isTypeOnly?: boolean + readonly isExternalLibraryImport?: boolean +} + + +interface DepGraphCache { + [file: string]: { + readonly hash: string + // readonly programs: string[] + readonly dependencies?: Dependency[] + } +} + +const incrementalFileName = `[#compile]__incremental__.json` +async function loadCache(fs: Pick = getProgramFs()): Promise { + try { + return JSON.parse(await fs.readFile(incrementalFileName, 'utf-8')) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + return {} + } +} + +async function saveCache(data: DepGraphCache, fs: Pick = getProgramFs()) { + await fs.writeFile(incrementalFileName, JSON.stringify(data)) +} + +function checkDidChange(k: string, cache: DepGraphCache, visited = new Set()) { + const cached = cache[k] + if (!cached) { + return true + } + + if (!cached.dependencies) { + return false + } + + for (const d of cached.dependencies) { + const f = d.fileName + if (!visited.has(f)) { + visited.add(f) + if (checkDidChange(f, cache, visited)) { + return true + } + } + } + + return false +} + +export type Compilation = { graph: CompilationGraph; changed: Set } + +export async function clearIncrementalCache() { + await getProgramFs().deleteFile(incrementalFileName).catch(throwIfNotFileNotFoundError) +} + +export const getFileHasher = memoize(() => createFileHasher(getFs(), getGlobalCacheDirectory())) + +// TODO: clear cache when updating packages +export type IncrementalHost = ReturnType +export function createIncrementalHost(opt: ts.CompilerOptions) { + const fileChecker = getFileHasher() + const checkFile = keyedMemoize(fileChecker.checkFile) + + const graphs = new WeakMap() + + async function getCachedGraph(oldHashes?: Record) { + const depsCache = await loadCache() + const invalidated = new Set() + + async function check(k: string, v: { hash: string; dependencies?: Dependency[] }) { + const r = await checkFile(k).catch(e => { + throwIfNotFileNotFoundError(e) + invalidated.add(k) + }) + + const h = oldHashes?.[k] ?? v.hash + if (h !== r?.hash || invalidated.has(k) || v.dependencies?.find(x => invalidated.has(x.fileName))) { + delete depsCache[k] + } + } + + await Promise.all(Object.entries(depsCache).map(([k, v]) => check(k, v))) + + return depsCache + } + + async function updateCachedGraph(cache: DepGraphCache, graph: CompilationGraph) { + const updatedCache: DepGraphCache = {} + const changed = new Set() + const p: Promise[] = [] + for (const sf of graph.files) { + if (cache[sf]) { + continue + } + + p.push(checkFile(sf).then(r => { + updatedCache[sf] = { hash: r.hash } + changed.add(sf) + }).catch(e => { + throwIfNotFileNotFoundError(e) + delete updatedCache[sf] + })) + } + + await Promise.all(p) + + for (let i = 0; i < graph.edges.length; i++) { + const [from, to] = graph.edges[i] + if (cache[from]) { + continue + } + + const o = updatedCache[from] + if (!o.dependencies) { + Object.assign(o, { dependencies: [] }) + } + o.dependencies!.push({ + fileName: to, + specifier: graph.specifiers.get(i)!, + isTypeOnly: graph.typeEdges.has(i) ? true : undefined + }) + } + + await Promise.all([ + saveCache({ ...cache, ...updatedCache }), + fileChecker.flush(), + ]) + + return changed + } + + async function getGraph(program: ts.Program, host?: ts.ModuleResolutionHost, cache?: DepGraphCache, roots = program.getRootFileNames()) { + if (graphs.has(program)) { + return graphs.get(program)! + } + + const opt = program.getCompilerOptions() + cache ??= await getCachedGraph() + + const getDeps = keyedMemoize(fileName => { + if (cache![fileName]) { + return cache![fileName].dependencies + } + + const sf = program.getSourceFile(fileName) + if (!sf) { + if (fileName.endsWith('.d.ts')) { + // getLogger().warn(`Missing external lib`, fileName) + return + } + + // This isn't always an error. The user could've deleted the file. + // Right now this only happens when we fail to evict a dependent file from the cache. + // throw new Error(`Missing source file: ${fileName}`) + return + } + + const deps = getDependencies(sf, opt, host) + + return deps + // .filter(d => d.resolved && !d.resolved.isExternalLibraryImport) + .filter(d => d.resolved) + .map(d => ({ + fileName: d.resolved!.resolvedFileName, + specifier: d.specifier, + isTypeOnly: d.isTypeOnly, + isExternalLibraryImport: d.resolved!.isExternalLibraryImport, + })) + }) + + const graph = getCompilationGraph(roots, getDeps) + const changed = await updateCachedGraph(cache, graph) + + graphs.set(program, { graph, changed }) + + return { graph, changed } + } + + async function _getTsCompilerHost() { + return ts.createCompilerHost(opt, true) + } + + const getTsCompilerHost = memoize(_getTsCompilerHost) + + async function getProgram(roots: string[], oldHashes?: Record) { + const host = await getTsCompilerHost() + const cache = await getCachedGraph(oldHashes) + const pruned = roots.filter(f => checkDidChange(f, cache)) + const program = ts.createProgram(pruned, opt, host) // This takes 99% of the time for `getProgram` + const { graph, changed } = await getGraph(program, host, cache, roots) + + return { program, graph, changed } + } + + async function getCachedDependencies(...files: string[]) { + const depsCache = await loadCache() + const graph = getCompilationGraph(files, f => depsCache[f]?.dependencies) + + return graph + } + + async function getCachedHashes() { + const depsCache = await loadCache() + const hashes: Record = {} + for (const [k, v] of Object.entries(depsCache)) { + hashes[k] = v.hash + } + + return hashes + } + + return { getProgram, getGraph, getCachedDependencies, getCachedHashes, getTsCompilerHost } +} + +export function getAllDependencies(graph: CompilationGraph, files: string[]) { + const deps = new Set() + const index: Record = {} + + for (let i = 0; i < graph.edges.length; i++) { + const [from, to] = graph.edges[i] + const arr = index[from] ??= [] + arr.push(to) + } + + function visit(k: string) { + if (deps.has(k)) { + return + } + + deps.add(k) + const z = index[k] + if (z) { + for (const d of z) { + visit(d) + } + } + } + + for (const f of files) { + visit(f) + } + + // Roots have to be determined by looking at the transitive dependencies + // of the original set of files rather than the entire graph + // + // IMPORTANT: cycles in the graph will be deleted by this method + // TODO: find the minimum feedback arc set + const roots = new Set(files) + for (const f of deps) { + const z = index[f] + if (z) { + for (const d of z) { + roots.delete(d) + } + } + } + + return { roots, deps } +} + + +export function getImmediateDependencies(graph: CompilationGraph, file: string) { + const deps = new Set() + + for (let i = 0; i < graph.edges.length; i++) { + const [from, to] = graph.edges[i] + if (from === file) { + deps.add(to) + } + } + + return deps +} diff --git a/src/compiler/programBuilder.ts b/src/compiler/programBuilder.ts new file mode 100644 index 0000000..1c0a1ab --- /dev/null +++ b/src/compiler/programBuilder.ts @@ -0,0 +1,591 @@ +import ts from 'typescript' +import * as path from 'node:path' +import type { TfJson } from 'synapse:terraform' +import { CompilerHost, CompilerOptions, Platform, readSources, synth } from './host' +import { JsonFs } from '../system' +import { createTransformer, getModuleBindingId, getTransformDirective } from './transformer' +import { SourceMapHost, getNullTransformationContext } from '../static-solver/utils' +import { createSchemaFactory } from './validation' +import { IncrementalHost, createIncrementalHost, getAllDependencies } from './incremental' +import { ResolvedProgramConfig, getOutputFilename, shouldInvalidateCompiledFiles } from './config' +import { getLogger, runTask } from '..' +import { createGraphCompiler, getModuleType } from '../static-solver' +import { ResourceTypeChecker, createResourceGraph } from './resourceGraph' +import { getWorkingDir } from '../workspaces' +import { getArtifactFs, getProgramFs } from '../artifacts' +import { getBuildTargetOrThrow } from '../execution' +import { compileZig } from '../zig/compile' +import { hasMainFunction, hasZigImport } from './entrypoints' +import { isWindows, makeRelative, resolveRelative } from '../utils' + +export interface ProgramBuilder { + emit(program: ts.Program, files?: string[]): Promise + synth(deployTarget: string): Promise +} + +export function createProgramBuilder( + config: ResolvedProgramConfig, + incrementalHost: IncrementalHost = createIncrementalHost(config.tsc.cmd.options), +) { + const sourcemapHost: SourceMapHost & ts.FormatDiagnosticsHost = { + getNewLine: () => ts.sys.newLine, + getCurrentDirectory: () => config.tsc.rootDir, + getCanonicalFileName: (ts as any).createGetCanonicalFileName(ts.sys.useCaseSensitiveFileNames) + } + + function getFiles() { + if (!config.csc.targetFiles) { + return + } + + const targetFiles = Array.isArray(config.csc.targetFiles) ? config.csc.targetFiles : [config.csc.targetFiles] + const resolved = targetFiles.map(f => path.resolve(config.csc.workingDirectory!, f)) + + return new Set(resolved) + } + + const targetFiles = getFiles() + + const rootDir = config.tsc.rootDir + const workingDir = getWorkingDir() + const outDir = config.tsc.cmd.options.outDir + + const shouldReplace = isWindows() + function normalizeFileName(f: string) { + return shouldReplace ? f.replaceAll('\\', '/') : f + } + + async function emit(program?: ts.Program, host?: ts.ModuleResolutionHost, incremental = config.csc.incremental) { + const afs = await getArtifactFs() + + if (!incremental && !config.csc.noInfra) { + afs.getCurrentProgramStore().clear('compile') + afs.getCurrentProgramStore().clear('declarations') + } + + if (shouldInvalidateCompiledFiles(config.tsc)) { + incremental = false + } + + // We probably don't need to load this during `watch`? + const oldSources = incremental ? await readSources() : undefined + const oldHashes = oldSources ? Object.fromEntries( + Object.entries(oldSources).map(([k, v]) => [path.resolve(workingDir, k), v.hash]) + ) : undefined + + // Disabling the incremental flag just means we won't try to use anything from a + // previous compilation. We will always persist the results regardless of `incremental` + if (incremental && !oldHashes) { + getLogger().log(`Performing full compile due to lack of previous compilation data`) + incremental = false + } + + if (!program) { + program = await runTask('compile', 'init program', async () => { + if (incremental) { + const res = await incrementalHost.getProgram(config.tsc.cmd.fileNames, oldHashes) + + return res.program + } + + return ts.createProgram(config.tsc.cmd.fileNames, config.tsc.cmd.options, await incrementalHost.getTsCompilerHost()) + }, 10) + } + + // The current overhead is about +60% vs. standard `tsc` + + // `lib.dom.d.ts` is pretty big, excluding it can save ~100ms on program init + // Removing all node types can save another ~200ms + // `lib.es5.d.ts` is 214k lines. Could get a decent speedup by replacing it with a minimal file + + // const largeFiles = program.getSourceFiles().map(x => [x.fileName, x.text.length] as const).sort((a, b) => b[1] - a[1]).slice(0, 10) + // const size = largeFiles.reduce((a, b) => a + b[1], 0) + // if (size >= 1_000_000) { + // getLogger().log(`Total size of top 10 largest files exceeds 1MB`, largeFiles) + // getLogger().log(config.tsc.cmd.options.types) + // } + + // `program.getTypeChecker()` is _not_ free. + // The typechecker doesn't do much if `noLib` is enabled, so we don't use it at all + const getTypeChecker = !config.tsc.cmd.options.noLib + ? () => runTask('', 'getTypeChecker', () => program.getTypeChecker(), 1) + : undefined + + const emitHost = createEmitHost() + const graphCompiler = createGraphCompiler(sourcemapHost, program.getCompilerOptions()) + const compilation = await runTask('compile', 'graph', () => incrementalHost.getGraph(program!, host), 1) + const resourceGraph = createResourceGraph(program, compilation, graphCompiler) + const newFiles = program.getSourceFiles() + .filter(x => !x.isDeclarationFile && !program.isSourceFileFromExternalLibrary(x)) + .map(x => normalizeFileName(x.fileName)) + + const allSourceFiles = new Set(config.tsc.files.filter(x => !!x.match(/\.tsx?$/)).map(normalizeFileName)) + + // ZIG COMPILATION + const isZigCompilationEnabled = !!(config.csc as any).zigFiles + const zigImportingFiles = isZigCompilationEnabled + ? new Map(await Promise.all([...allSourceFiles].map(async x => [x, await hasZigImport(path.relative(workingDir, x))] as const))) + : undefined + + if (isZigCompilationEnabled) { + const includedZigFiles = new Set((config.csc as any).zigFiles) + for (const f of includedZigFiles) { + const z = await compileZig(path.resolve(workingDir, f), config) + if (z) { + getLogger().log('Compiled zig file', z) + } + } + } + + const changed = compilation.changed + + // We should not do any type analysis until installing package deps (if any) + await runTask('compile', 'types', async () => { + const changedFiles = !incremental ? newFiles : newFiles.filter(x => changed.has(x)) + await resourceGraph.generateTypes(changedFiles, config.tsc.rootDir, config.tsc.cmd.options, incremental) + }, 1) + + const infraFiles = new Set() + const executables = new Set() + const needsRuntimeTransform = new Set() + + if ((config.csc as any).forcedInfra) { + (config.csc as any).forcedInfra.forEach((f: string) => { + getLogger().debug(`Marked ${f} as infra file [forced]`) + infraFiles.add(path.resolve(workingDir, f)) + }) + } + + const oldEntrypoints = incremental ? await getEntrypointsFile() : undefined + + function isDeclarationFile(fileName: string) { + if (fileName.endsWith('.d.ts')) { + return true + } + + if (!config.tsc.cmd.options.allowArbitraryExtensions) { + return false + } + + return !!fileName.match(/\.d\.([^\.]+)\.ts$/) + } + + function determineCompilationModes(program: ts.Program) { + for (const f of allSourceFiles) { + if (isDeclarationFile(f)) continue + + const r = resourceGraph.getFileResourceInstantiations(f).length + if (r > 0) { + getLogger().debug(`Marked ${f} as infra file`) + infraFiles.add(f) + } else if (resourceGraph.hasCalledCallables(f) || zigImportingFiles?.get(f)) { + getLogger().debug(`Marked ${f} for runtime transforms`) + needsRuntimeTransform.add(f) + } + + if (!incremental) { + if (hasMainFunction(getSourceFileOrThrow(program, f), getTypeChecker)) { + getLogger().debug(`Marked ${f} as an executable`) + executables.add(f) + } + } else { + const relPath = path.relative(getWorkingDir(), f) + if (changed.has(f)) { + if (hasMainFunction(getSourceFileOrThrow(program, f), getTypeChecker)) { + getLogger().debug(`Marked ${f} as an executable`) + executables.add(f) + } + } else if (oldEntrypoints?.executables?.[relPath]) { + executables.add(f) + } + } + } + } + + runTask('compile', 'mode analysis', () => determineCompilationModes(program), 10) + + const compilerHost = createHost(graphCompiler, resourceGraph, sourcemapHost, program, config.csc) + + const isShared = config.csc.sharedLib + const allDeps = infraFiles.size > 0 && !isShared + ? getAllDependencies(compilation.graph, [...infraFiles]) + : undefined + + // Entrypoints for synthesis, not package entrypoints + const entrypoints = allDeps ? [...allDeps.roots].map(x => path.relative(getWorkingDir(), x)) : [] + const deployables = Object.fromEntries( + [...infraFiles].map(f => [path.relative(getWorkingDir(), f), getOutputFilename(config.tsc.rootDir, config.tsc.cmd.options, f)]) + ) + + const entrypointsFile: EntrypointsFile = { + entrypoints, + deployables, + executables: Object.fromEntries( + [...executables].map(f => [path.relative(getWorkingDir(), f), getOutputFilename(config.tsc.rootDir, config.tsc.cmd.options, f)]) + ) + } + + emitHost.emitEntrypointsFile(entrypointsFile) + + const compiledFiles = new Set() + const declaration = config.tsc.cmd.options.declaration + + function getCompiledEntrypointsSet() { + if (declaration !== undefined || !config.compiledEntrypoints) { + return + } + + const resolved = config.compiledEntrypoints.map(x => resolveRelative(getWorkingDir(), x)) + + return getAllDependencies(compilation.graph, resolved).deps + } + + const compiledEntrypointsSet = getCompiledEntrypointsSet() + + function shouldEmitDeclaration(sourceFile: string) { + if (declaration !== undefined) { + return declaration + } + + if (!compiledEntrypointsSet) { + return false + } + + return compiledEntrypointsSet.has(sourceFile) + } + + function doCompile(program: ts.Program) { + for (const f of allSourceFiles) { + if (isDeclarationFile(f)) { + continue + } + + if (targetFiles && !targetFiles.has(f)) { + continue + } + + if (incremental && !changed.has(f)) { + getLogger().debug(`Skipped unchanged source file ${f}`) + continue + } + + const sf = getSourceFileOrThrow(program, f) + const writeOpt: WriteCallbackOptions = !isShared + ? getWriteCallbackOptions(sf, allDeps?.deps.has(f) ?? false, config.csc.includeJs, shouldEmitDeclaration(f), infraFiles?.has(f), needsRuntimeTransform.has(f)) + : { + applyTransform: true, + saveTransform: true, + declaration, + } + + if (config.csc.noInfra) { + emitHost.emitNoInfra(program, compilerHost, sf, writeOpt) + } else { + emitHost.emit(program, compilerHost, sf, writeOpt) + } + + compiledFiles.add(f) + } + } + + runTask('compile', 'transform', () => doCompile(program), 10) + + function getIncluded() { + return config.tsc.files.filter(f => !allSourceFiles.has(f) || f.endsWith('.d.ts')) + } + + // note: `watch` needs to be treated the same as `incremental` + await runTask('compile', 'emit', async () => { + await emitHost.complete(compilerHost, rootDir, outDir, getIncluded(), config.csc) + // if (!config.csc.noInfra) { + // await compilerHost.finish(incremental) + // } + }, 1) + + return { + compilation, + infraFiles, + compiledFiles, + entrypointsFile, + } + } + + async function _synth(deployTarget: string, entrypointsFile?: EntrypointsFile) { + if (!entrypointsFile) { + entrypointsFile = await getEntrypointsFile() + if (!entrypointsFile || entrypointsFile.entrypoints.length === 0) { + throw new Error(`No entrypoints`) + } + } + + const bt = getBuildTargetOrThrow() + + const sources = await readSources() // double read + if (!sources) { + throw new Error(`No compilation artifacts found`) + } + + const mapped = Object.entries(sources).filter(([_, v]) => !v.isTsArtifact).map(([k, v]) => ({ + name: path.resolve(bt.workingDirectory, v.outfile), + source: path.resolve(bt.workingDirectory, k) + })) + + return synth(entrypointsFile.entrypoints, Object.values(entrypointsFile.deployables), { + outDir, + sources: mapped, + compilerOptions: { ...config.csc, deployTarget }, + + esm: getModuleType(config.tsc.cmd.options.module) === 'esm', + }) + } + + async function printTypes(target?: string) { + const program = await runTask('compile', 'init program', async () => { + const host = ts.createCompilerHost(config.tsc.cmd.options, true) + + return ts.createProgram(config.tsc.cmd.fileNames, config.tsc.cmd.options, host) + }, 100) + + const graphCompiler = createGraphCompiler(sourcemapHost, program.getCompilerOptions()) + const compilation = await incrementalHost.getGraph(program) + const resourceGraph = createResourceGraph(program, compilation, graphCompiler) + const newFiles = program.getSourceFiles().filter(x => !x.isDeclarationFile).map(x => x.fileName) + await runTask('compile', 'init types', async () => { + await resourceGraph.generateTypes(newFiles, config.tsc.rootDir, config.tsc.cmd.options, false) + }, 100) + + for (const f of newFiles) { + resourceGraph.printTypes(f) + } + } + + return { emit, synth: _synth, printTypes } +} + +interface EntrypointsFile { + entrypoints: string[] + + // The next two fields map relative source file name to abs. output file + deployables: Record + executables: Record +} + +function makeEntrypointsRelative(data: EntrypointsFile, workingDir = getWorkingDir()) { + const relative = { ...data, deployables: { ...data.deployables }, executables: { ...data.executables } } + for (const [k, v] of Object.entries(relative.deployables)) { + relative.deployables[k] = makeRelative(workingDir, v) + } + for (const [k, v] of Object.entries(relative.executables)) { + relative.executables[k] = makeRelative(workingDir, v) + } + return relative +} + +function resolveEntrypointsFile(data: EntrypointsFile, workingDir = getWorkingDir()) { + for (const [k, v] of Object.entries(data.deployables)) { + data.deployables[k] = resolveRelative(workingDir, v) + } + for (const [k, v] of Object.entries(data.executables)) { + data.executables[k] = resolveRelative(workingDir, v) + } + return data +} + +async function setEntrypointsFile(data: EntrypointsFile) { + const fs = getProgramFs() + await fs.writeJson(`[#compile]__entrypoints__.json`, makeEntrypointsRelative(data)) +} + +export async function getEntrypointsFile(fs: Pick = getProgramFs()): Promise { + const data = await fs.readJson(`[#compile]__entrypoints__.json`).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + }) + + return data ? resolveEntrypointsFile(data) : undefined +} + +export async function getExecutables() { + const f = await getEntrypointsFile() + + return f?.executables +} + +export async function getDeployables() { + const f = await getEntrypointsFile() + + return f?.deployables +} + +interface WriteCallbackOptions { + readonly moduleId?: string + readonly includeJs?: boolean + readonly needsDeploy?: boolean + readonly declaration?: boolean + readonly saveTransform?: boolean + readonly applyTransform?: boolean + readonly applyRuntimeTransform?: boolean + readonly internalModule?: boolean +} + +function isInternal(sourceFile: ts.SourceFile) { + return sourceFile.getFullText().startsWith('//@internal') +} + +function getWriteCallbackOptions(sourceFile: ts.SourceFile, isInfra: boolean, includeJs?: boolean, declaration?: boolean, needsDeploy?: boolean, applyRuntimeTransform?: boolean): WriteCallbackOptions { + const moduleId = getModuleBindingId(sourceFile) + const isBoundModule = moduleId !== undefined + const shouldPersist = getTransformDirective(sourceFile) === 'persist' + const shouldTransform = shouldPersist || (!isBoundModule && isInfra) + const shouldIncludeJs = (!needsDeploy && includeJs) || isBoundModule + + return { + moduleId, + needsDeploy, + declaration, + applyRuntimeTransform, + // Missing package entry: dist/src/zig/util.zig [resolving ./util.zig from dist/src/zig/util.js] + // applyRuntimeTransform: applyRuntimeTransform && !isInfra, + applyTransform: shouldTransform, + saveTransform: shouldPersist, + includeJs: shouldIncludeJs, + internalModule: !!moduleId && isInternal(sourceFile), + } +} + +export function createEmitHost() { + const filePromises: Promise[] = [] + + function emitWorker(compilerHost: CompilerHost, fileName: string, text: string, sourceFile: ts.SourceFile, opt: WriteCallbackOptions) { + if (fileName.endsWith('.js')) { + if (opt.applyTransform) { + compilerHost.compileSourceFile(ts.getOriginalNode(sourceFile, ts.isSourceFile)) + } + + return compilerHost.saveTscFile(sourceFile, fileName, text, opt.moduleId, opt.internalModule) + } else if (fileName.endsWith('.d.ts') || fileName.endsWith('.d.ts.map') || fileName.endsWith('.js.map')) { + return compilerHost.saveTscFile(sourceFile, fileName, text) + } + + throw new Error(`Unknown file output "${fileName}"`) + } + + // Maybe not needed anymore. + function emitMissingIncluded(outDir: string, rootDir: string, include: string[]) { + const fs = getProgramFs() + filePromises.push(...include.map(async f => { + const outputPath = path.resolve( + outDir, + path.relative(rootDir, f) + ) + return fs.writeFile(`[#compile]${outputPath}`, await fs.readFile(f)) + })) + } + + function createWriteCallback(compilerHost: CompilerHost, sf: ts.SourceFile, opt: WriteCallbackOptions) { + function writeCallback(fileName: string, text: string, writeByteOrderMark: boolean, onError?: (message: string) => void) { + try { + filePromises.push(emitWorker(compilerHost, fileName, text, sf, opt)) + } catch (e) { + onError?.((e as any).message) + } + } + + return writeCallback + } + + function emit(program: ts.Program, compilerHost: CompilerHost, sourceFile: ts.SourceFile, opt: WriteCallbackOptions) { + const writeCallback = createWriteCallback(compilerHost, sourceFile, opt) + if (opt.includeJs && !opt.saveTransform && !opt.applyRuntimeTransform) { + const r = program.emit(sourceFile, writeCallback) + if (r.emitSkipped || r.diagnostics.length > 0) { + // TODO: fail here? + getLogger().warn(...r.diagnostics.map(x => ts.formatDiagnostic(x, compilerHost.sourceMapHost as any))) + } else { + const bt = getBuildTargetOrThrow() + compilerHost.addSource( + path.relative(bt.workingDirectory, sourceFile.fileName), + compilerHost.getOutputFilename(sourceFile.fileName), + true + ) + } + } else { + if (opt.declaration) { + const result = program.emit(sourceFile, writeCallback, undefined, true) + if (result.emitSkipped || result.diagnostics.length > 0) { + // TODO: fail here? + getLogger().warn(...result.diagnostics.map(x => ts.formatDiagnostic(x, compilerHost.sourceMapHost as any))) + } + } + + if (opt.applyRuntimeTransform) { + compilerHost.compileSourceFileRuntimeOnly(sourceFile) + } else { + if (opt.needsDeploy) { + filePromises.push(compilerHost.emitDeployStub(sourceFile)) + } + compilerHost.compileSourceFile(sourceFile, opt.saveTransform, opt.moduleId, opt.internalModule) + } + } + } + + function emitNoInfra(program: ts.Program, compilerHost: CompilerHost, sourceFile: ts.SourceFile, opt: WriteCallbackOptions) { + filePromises.push(compilerHost.emitNoInfra(sourceFile, opt.needsDeploy)) + } + + function emitEntrypointsFile(data: EntrypointsFile) { + filePromises.push(setEntrypointsFile(data)) + } + + async function complete(compilerHost: CompilerHost, rootDir: string, outDir?: string, include?: string[], config?: { noInfra?: boolean; incremental?: boolean }) { + if (outDir && include) { + emitMissingIncluded(outDir, rootDir, include) + } + + if (!config?.noInfra) { + filePromises.push(compilerHost.finish(config?.incremental)) + } + + await Promise.all(filePromises) + filePromises.length = 0 + } + + return { createWriteCallback, emit, emitNoInfra, emitEntrypointsFile, complete } +} + +function getSourceFileOrThrow(program: Pick, fileName: string) { + const sf = program.getSourceFile(fileName) + if (!sf) { + throw new Error(`No source file found: ${fileName}`) + } + return sf +} + +export function createHost( + graphCompiler: ReturnType, + resourceTypeChecker: ResourceTypeChecker, + sourceMapHost: SourceMapHost, + program: ts.Program, + options?: CompilerOptions +) { + const schemaFactory = createSchemaFactory(program) + const tsOptions = program.getCompilerOptions() + const resourceTransformer = createTransformer( + getWorkingDir(), + getNullTransformationContext(), + graphCompiler, + schemaFactory, + resourceTypeChecker, + ) + + return new CompilerHost( + sourceMapHost, + graphCompiler, + resourceTransformer, + resourceTypeChecker, + tsOptions, + options, + ) +} + diff --git a/src/compiler/resourceGraph.ts b/src/compiler/resourceGraph.ts new file mode 100644 index 0000000..30683be --- /dev/null +++ b/src/compiler/resourceGraph.ts @@ -0,0 +1,999 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { CompilerOptions } from './host' +import { createSyntheticComment, failOnNode, getNodeLocation, isExported, isWindows, keyedMemoize, makeRelative, resolveRelative, sortRecord, throwIfNotFileNotFoundError } from '../utils' +import { JsonFs } from '../system' +import { createGraphCompiler, Scope, Symbol } from '../static-solver' +import { Compilation } from './incremental' +import { getLogger } from '..' +import { getCallableDirective, getResourceDirective } from './transformer' +import { getProgramFs } from '../artifacts' +import { getWorkingDir } from '../workspaces' +import { loadTypes } from '../pm/packages' +import { getOutputFilename } from './config' +import { printLine } from '../cli/ui' +import { annotateNode, compilerError } from './diagnostics' + +interface SymbolNameComponents { + name?: string + fileName?: string + specifier?: string + isImported?: boolean +} + +interface ResourceInstantiation { + readonly kind: string // FQN/Symbol + +} + +export interface TypeInfo { + intrinsic?: boolean + instanceType?: string + callable?: string // the callable member on the _instance_ + // this describes what resources _would_ be created if the type is instantiated + instantiations?: ResourceInstantiation[] + + // Used for property/element access expressions + memberName?: string + members?: Record +} + +export interface TypesFileData { + [fileName: string]: Record +} + +// Used for incremental builds +interface StoredTypesFilesData { + [sourceFileName: string]: { + outfile: string + exports: Record + instantiations: ResourceInstantiation[] + hasCalledCallable: boolean + } +} + +export type ResourceTypeChecker = ReturnType +export function createResourceGraph( + program: ts.Program, + compilation: Compilation, + graphCompiler: ReturnType +) { + const importMap = new Map>() + for (let i = 0; i < compilation.graph.edges.length; i++) { + if (compilation.graph.typeEdges.has(i)) { + continue + } + + const edge = compilation.graph.edges[i] + const spec = compilation.graph.specifiers.get(i) + if (!spec) { + throw new Error(`Missing specifier from "${edge[0]}" to "${edge[1]}"`) + } + + const m = importMap.get(edge[0]) ?? new Map() + importMap.set(edge[0], m) + m.set(spec, edge[1]) + } + + const runtimeModulesDecls = new Map() + const reverseRuntimeModulesDecls = new Map() + + function getFilePathFromSpecifier(spec: string, origin: string) { + const m = importMap.get(origin) + // if (!m) { + // throw new Error(`No import mapping found for file: ${origin}`) + // } + + return m?.get(spec) ?? runtimeModulesDecls.get(spec) + } + + const fileSymbols = new Map>() + function getFileSymbols(fileName: string) { + // XXX: windows hack + if (isWindows()) { + fileName = fileName.replaceAll('\\', '/') + } + if (fileSymbols.has(fileName)) { + return fileSymbols.get(fileName)! + } + + const checker = getSourceFileTypeChecker(fileName) + fileSymbols.set(fileName, checker.exported) + checker.init() + + return checker.exported + } + + const fileInstantiations = new Map() + function getFileResourceInstantiations(fileName: string) { + // XXX: windows hack + if (isWindows()) { + fileName = fileName.replaceAll('\\', '/') + } + if (fileInstantiations.has(fileName)) { + return fileInstantiations.get(fileName)! + } + + const checker = getSourceFileTypeChecker(fileName) + const insts = checker.getResourceInstantiations() + fileInstantiations.set(fileName, insts) + + return insts + } + + const calledCallables = new Map() + function hasCalledCallables(fileName: string) { + // XXX: windows hack + if (isWindows()) { + fileName = fileName.replaceAll('\\', '/') + } + if (calledCallables.has(fileName)) { + return calledCallables.get(fileName)! + } + + const checker = getSourceFileTypeChecker(fileName) + return checker.hasCalledCallables() + } + + function toString(components: SymbolNameComponents) { + // Note that using the specifier isn't correct unless we qualify the origin + const moduleName = components.specifier ?? components.fileName + if (!components.name && !moduleName) { + throw new Error(`Symbol components contain no module name or symbol name: ${JSON.stringify(components, undefined, 4)}`) + } + + return moduleName ? `"${moduleName}"${components.name ? `.${components.name}` : ''}` : components.name! + } + + function getImportName(sym: Symbol, clause: ts.ImportClause) { + const parent = sym.declaration?.parent + if (parent === clause) { + return 'default' + } + + if (parent && ts.isImportSpecifier(parent)) { + const name = parent.propertyName ?? parent.name + + return name.text + } + } + + const shouldReplace = isWindows() + function normalizeFileName(f: string) { + return shouldReplace ? f.replaceAll('\\', '/') : f + } + + function resolveModuleSpecifier(node: ts.Node) { + const origin = normalizeFileName(node.getSourceFile().fileName) + const moduleSpec = (node as ts.StringLiteral).text + + return { + specifier: moduleSpec, + fileName: getFilePathFromSpecifier(moduleSpec, origin), + } + } + + function getNameComponents(sym: Symbol): SymbolNameComponents { + if (sym.parent) { + const parentFqn = getNameComponents(sym.parent) + + return { + ...parentFqn, + name: parentFqn.name ? `${parentFqn.name}.${sym.name}` : sym.name, + } + } + + if (sym.importClause) { + const { fileName, specifier } = resolveModuleSpecifier(sym.importClause.parent.moduleSpecifier) + const name = getImportName(sym, sym.importClause) + + return { fileName, specifier, name, isImported: true } + } + + const fileName = sym.declaration?.getSourceFile().fileName + + return { fileName: fileName ? normalizeFileName(fileName) : undefined, name: sym.name } + } + + function getNameComponentsFromNode(node: ts.Node): SymbolNameComponents | undefined { + if (node.kind === ts.SyntaxKind.CallExpression || node.kind === ts.SyntaxKind.NewExpression) { + return getNameComponentsFromNode((node as ts.CallExpression).expression) + } else if (node.kind === ts.SyntaxKind.AwaitExpression || node.kind === ts.SyntaxKind.ParenthesizedExpression) { + return getNameComponentsFromNode((node as ts.AwaitExpression).expression) + } + + const sym = graphCompiler.getSymbol(node) + + return sym ? getNameComponents(sym) : undefined + } + + function isDeferCall(components: SymbolNameComponents) { + return components?.specifier === 'synapse:core' && components.name === 'defer' + } + + function isUsingCall(components: SymbolNameComponents) { + return components?.specifier === 'synapse:core' && components.name === 'using' + } + + function isDefineResource(node: ts.Expression) { + if (!ts.isCallExpression(node)) { + return false + } + + const components = getNameComponentsFromNode(node.expression) + + return components?.specifier === 'synapse:core' && components.name === 'defineResource' + } + + function isDefineDataSource(node: ts.Expression) { + if (!ts.isCallExpression(node)) { + return false + } + + const components = getNameComponentsFromNode(node.expression) + + return components?.specifier === 'synapse:core' && components.name === 'defineDataSource' + } + + function getExternalNodeType(name: string, components: SymbolNameComponents) { + const module = components.specifier + if (module && (module.startsWith('synapse-provider:') || module === 'synapse:lib')) { + return { intrinsic: true } + } + + if (module === 'synapse:core' && name === 'getBuildTarget') { + return { intrinsic: true } + } + + if (!components.fileName) { + return + } + + const symbols = getFileSymbols(components.fileName) + + return symbols[name] + } + + function isFunctionLikeDeclaration(node: ts.Node): node is ts.FunctionLikeDeclaration { + return ts.isFunctionExpression(node) || ts.isArrowFunction(node) || ts.isFunctionDeclaration(node) || ts.isConstructorDeclaration(node) || ts.isMethodDeclaration(node) + } + + interface TypeChecker { + readonly exported: Record + getNodeType(node: ts.Node): TypeInfo | undefined + } + + function _getSourceFileTypeChecker(fileName: string) { + const sf = program.getSourceFile(fileName) + if (!sf) { + throw new Error(`No source file: ${fileName}`) + } + + return createSourceFileTypeChecker(sf) + } + + const getSourceFileTypeChecker = keyedMemoize(_getSourceFileTypeChecker) + + function createSourceFileTypeChecker(sf: ts.SourceFile) { + const exported: Record = {} + const stack: Map[] = [new Map()] + const types = new Map>() + const symbolTypes = new Map>() + const calledCallables = new Map() + + function addInstantiations(node: ts.Node, ...instances: ResourceInstantiation[]) { + const instantiations = stack[stack.length - 1] + instantiations.set(node, instances) + } + + function getSymbolType(sym: Symbol) { + if (symbolTypes.has(sym)) { + return symbolTypes.get(sym) + } + + const type: Partial = {} + symbolTypes.set(sym, type) + + if (!sym.variableDeclaration?.initializer) { + return + } + + const initializer = sym.variableDeclaration.initializer + if (isDefineResource(initializer) || isDefineDataSource(initializer)) { + type.intrinsic = true + + return type + } + + const initType = visitExpression(initializer) + if (initType?.instanceType) { + type.callable = initType.callable + type.instanceType = initType.instanceType + type.members = initType.members + + return type + } + + if (initType) { + type.callable = initType.callable + if (initType.intrinsic || (initType.instantiations && initType.instantiations.length > 0)) { + const components = getNameComponentsFromNode(initializer) + if (!components) { + failOnNode(`Missing symbol for node`, initializer) + } + + type.instanceType = toString(components) + type.members = initType.members + } + + return type + } + } + + function getNodeType(node: ts.Node): Partial | undefined { + if (types.has(node)) { + return types.get(node) + } + + if (ts.isNewExpression(node) || ts.isCallExpression(node)) { + const type: Partial = visitExpression(node) ?? {} + types.set(node, type) + + return type + } + + const type: Partial = {} + types.set(node, type) + + if (ts.isVariableDeclaration(node)) { + const sym = graphCompiler.getSymbol(node) + if (!sym) { + getLogger().warn('missing symbol', getNodeLocation(node)) + return + } + + const type = getSymbolType(sym) + if (!type) { + return + } + + types.set(node, type) + + return type + } + + const isClassLike = ts.isClassLike(node) + if (!isClassLike && !isFunctionLikeDeclaration(node)) { + return + } + + const resourceDirective = getResourceDirective(node) + if (resourceDirective) { + const callable = getCallableDirective(node) + type.intrinsic = true + type.callable = callable + + if (isClassLike) { + const methods: Record = {} + for (const m of node.members) { + if (m.kind === ts.SyntaxKind.MethodDeclaration && getResourceDirective(m)) { + methods[m.name!.getText()] = { intrinsic: true } + } + } + type.members = methods + } + + return type + } + + stack.push(new Map()) + + const declType = isClassLike + ? visitClassLikeDeclaration(node) + : visitFunctionLikeDeclaration(node) + + const instantiations = stack.pop()! + type.intrinsic = declType?.intrinsic + + const insts = [...instantiations.values()].flat() + if (insts.length > 0) { + type.instantiations = insts + } + + return type + } + + function checkNode(node: ts.Node) { + const components = getNameComponentsFromNode(node) + if (!components || !components.name) { + return + } + + if (isDeferCall(components)) { + return getNodeType((node.parent as ts.CallExpression).arguments[0]) + } + + if (isUsingCall(components)) { + return getNodeType((node.parent as ts.CallExpression).arguments[1]) + } + + const isLocal = components.fileName === normalizeFileName(node.getSourceFile().fileName) + if (!isLocal) { + return getExternalNodeType(components.name, components) + } + + const sym = graphCompiler.getSymbol(node) + if (!sym) { + return + } + + if (sym.parent) { + const ty = getSymbolType(sym.parent) + const memberType = ty?.members?.[sym.name] + if (sym.parent.name !== 'this' || memberType) { // Need to implement types for `this` + return memberType + } + } + + const valueDecl = sym?.declaration + if (!valueDecl) { + return + } + + return getNodeType(valueDecl) + } + + function visitExpression(node: ts.Expression): TypeInfo | undefined { + if (ts.isCallExpression(node) || ts.isNewExpression(node)) { + if (node.arguments) { + node.arguments.forEach(visitExpression) + } + + const type = checkNode(node.expression) + if (type?.intrinsic) { + const kind = toString(getNameComponentsFromNode(node.expression)!) + addInstantiations(node, { kind }) + + return { instanceType: kind, callable: type.callable, members: type.members } + } + + if (type?.instantiations) { + addInstantiations(node, ...type.instantiations) + } + + if (type?.callable) { + calledCallables.set(node, type) + } + + return type + } else if (ts.isFunctionExpression(node) || ts.isClassExpression(node) || ts.isArrowFunction(node)) { + return getNodeType(node) + } else if (ts.isAwaitExpression(node) || ts.isParenthesizedExpression(node)) { + return visitExpression(node.expression) + } else if (ts.isPropertyAccessExpression(node)) { + const ty = checkNode(node.expression) + + return ty?.members?.[node.name.text] + } else { + node.forEachChild(visit) + } + } + + function visitStatements(nodes: readonly ts.Statement[]) { + const remainder: ts.Statement[] = [] + for (const n of nodes) { + // TODO: these should be visited in the order of their dependencies + if (ts.isClassDeclaration(n) || ts.isFunctionDeclaration(n)) { + getNodeType(n) + } else { + remainder.push(n) + } + } + + for (const n of remainder) { + if (ts.isExpressionStatement(n)) { + visitExpression(n.expression) + } else { + n.forEachChild(visit) + } + } + } + + function visitFunctionLikeDeclaration(node: ts.FunctionLikeDeclaration) { + for (const p of node.parameters) { + if (p.initializer) { + visitExpression(p.initializer) + } + } + + if (node.body) { + if (ts.isExpression(node.body)) { + visitExpression(node.body) + } else { + visitStatements(node.body.statements) + } + } + } + + function visitClassLikeDeclaration(node: ts.ClassLikeDeclaration) { + for (const m of node.members) { + if (ts.isPropertyDeclaration(m) && m.initializer) { + visitExpression(m.initializer) + } + + if (ts.isConstructorDeclaration(m)) { + visitFunctionLikeDeclaration(m) + } + + // TODO: static block? + } + + const superClass = node.heritageClauses?.find(x => x.token === ts.SyntaxKind.ExtendsKeyword)?.types[0].expression + if (superClass) { + if (isDefineResource(superClass)) { + return { intrinsic: true } + } else { + if (ts.isIdentifier(superClass) || ts.isExpressionWithTypeArguments(superClass)) { + const ty = checkNode(!ts.isIdentifier(superClass) ? superClass.expression : superClass) + if (ty?.instantiations) { + addInstantiations(node, ...ty.instantiations) + } + } else { + visitExpression(superClass) + } + } + } + } + + function visit(node: ts.Node) { + if (ts.isExpression(node)) { + visitExpression(node) + } else { + node.forEachChild(visit) + } + } + + // XXX: this is a hack to allow for late binding + function init() { + function visitExported(node: ts.Node) { + const type = getNodeType(node) + if (type?.intrinsic || (type?.instantiations && type.instantiations.length > 0) || type?.instanceType) { + exported[getNameComponentsFromNode(node)!.name!] = type + } + } + + for (const s of sf.statements) { + if (ts.isExportDeclaration(s) && !s.isTypeOnly) { + if (!s.moduleSpecifier) { + if (!s.exportClause || !ts.isNamedExports(s.exportClause)) { + continue + } + + for (const spec of s.exportClause.elements) { + const exportName = spec.name.text + const key = spec.propertyName ?? spec.name + const type = checkNode(key) + if (type) { + exported[exportName] = type + } + } + + continue + } + + const fileName = resolveModuleSpecifier(s.moduleSpecifier).fileName + if (!fileName) { + continue + } + + const symbols = getFileSymbols(fileName) + if (!s.exportClause) { + for (const [k, v] of Object.entries(symbols)) { + exported[k] = v + } + } else if (ts.isNamedExports(s.exportClause)) { + for (const spec of s.exportClause.elements) { + const exportName = spec.name.text + const key = spec.propertyName?.text ?? exportName + exported[exportName] = symbols[key] + } + } else if (ts.isNamespaceExport(s.exportClause)) { + for (const [k, v] of Object.entries(symbols)) { + exported[`${s.exportClause.name.text}.${k}`] = v + } + } + + continue + } + + if (!isExported(s)) { + if (ts.isVariableStatement(s)) { + for (const decl of s.declarationList.declarations) { + getNodeType(decl) + } + } + continue + } + + if (ts.isVariableStatement(s)) { + for (const decl of s.declarationList.declarations) { + visitExported(decl) + } + } else { + visitExported(s) + } + } + } + + function infoToComments(info: Partial) { + const comments: ts.SynthesizedComment[] = [] + if (info.instantiations) { + comments.push(...info.instantiations.map(x => createSyntheticComment(` ${x.kind}`))) + } + + return comments + } + + function printTypes() { + // function visit(node: ts.Node) { + // const info = getNodeType(node) + // if (info) { + // ts.setSyntheticLeadingComments(node, infoToComments(info)) + // } + + // node.forEachChild(visit) + // } + + // visit(sf) + + for (const [k, v] of types) { + if (v.callable || v.instanceType || v.intrinsic || v.instantiations) { + printLine(getNodeLocation(k), JSON.stringify(v, undefined, 4)) + } + } + + // getLogger().log(printNodes(sf.statements, sf)) + } + + function getResourceInstantiations() { + const instantiations = new Map() + function visitNode(node: ts.Node) { + const type = getNodeType(node) + if ((type?.instantiations && type.instantiations.length > 0) || type?.instanceType) { + const inst = [...type.instantiations ?? []] + if (type.instanceType) { + inst.push({ kind: type.instanceType }) + } + instantiations.set(node, inst) + } + } + + for (const s of sf.statements) { + if (ts.isVariableStatement(s)) { + for (const decl of s.declarationList.declarations) { + visitNode(decl) + } + } else if (ts.isExpressionStatement(s)) { + visitNode(s.expression) + } else if (!ts.isFunctionDeclaration(s) && !ts.isClassDeclaration(s)) { + visitNode(s) + } + } + + // for (const [k, v] of instantiations) { + // console.log(getNodeLocation(k), v) + // } + + return [...instantiations.values()].flat() + } + + function hasCalledCallables() { + return calledCallables.size > 0 + } + + getMarkedNodes(sf) + + return { + exported, + init, + getNodeType, + getSymbolType, + printTypes, + getResourceInstantiations, + hasCalledCallables, + } + } + + function registerTypes(dir: string, data: TypesFileData) { + getLogger().log(`Registered types for: "${dir}"`) + + for (const [k, v] of Object.entries(data)) { + fileSymbols.set(resolveRelative(dir, k), v) + } + } + + async function generateTypes(files: string[], tscRootDir: string, opt: ts.CompilerOptions, incremental = false) { + const { types, runtimeModules } = await loadTypes() + + for (const [k, v] of Object.entries(runtimeModules)) { + runtimeModulesDecls.set(k, v) + reverseRuntimeModulesDecls.set(v, k) + } + + for (const [k, v] of Object.entries(types)) { + registerTypes(k, v) + } + + const workingDirectory = getWorkingDir() + const outfiles = Object.fromEntries(files.map(fileName => { + const outfile = getOutputFilename(tscRootDir, opt, fileName) + + return [ + fileName, + makeRelative(workingDirectory, outfile).replace(/\.js$/, '.d.ts') + ] + })) + + const data: StoredTypesFilesData = {} + + if (incremental) { + const oldTypes = await getTypes() + for (const [k, v] of Object.entries(oldTypes ?? {})) { + const resolved = resolveRelative(workingDirectory, k) + if (!outfiles[resolved]) { + data[k] = v + fileSymbols.set(resolved, v.exports) + fileInstantiations.set(resolved, v.instantiations) + calledCallables.set(resolved, v.hasCalledCallable) + } + } + } + + for (const fileName of files) { + const outfile = outfiles[fileName] + data[makeRelative(workingDirectory, fileName)] = { + outfile, + exports: getFileSymbols(fileName), + instantiations: getFileResourceInstantiations(fileName), + hasCalledCallable: hasCalledCallables(fileName), + } + } + + + await setTypes(data) + } + + function makeTypesRelative(data: StoredTypesFilesData) { + for (const v of Object.values(data)) { + for (const info of Object.values(v.exports)) { + if (info.instanceType) { + info.instanceType = makeRelativeId(info.instanceType) + } + + if (info.instantiations) { + info.instantiations = info.instantiations.map(inst => ({ ...inst, kind: makeRelativeId(inst.kind) })) + } + } + + v.instantiations = v.instantiations.map(inst => ({ ...inst, kind: makeRelativeId(inst.kind) })) + } + } + + function makeRelativeId(typeId: string) { + const [_, f, n] = typeId.match(/^"([^"]+)"\.(.+)$/) ?? [] + if (!f) { + return typeId + } + + const spec = reverseRuntimeModulesDecls.get(f) + if (spec) { + return `"${spec}".${n}` + } + + return typeId + } + + function getCallableMemberName(sym: Symbol) { + const components = getNameComponents(sym) + if (!components.fileName) { + return + } + + if (components.isImported) { + if (!components.name) { + return + } + + return getFileSymbols(components.fileName)[components.name]?.callable + } + + const checker = getSourceFileTypeChecker(components.fileName) + const type = checker.getSymbolType(sym) + + return type?.callable + } + + function getNodeType(node: ts.Node) { + const components = getNameComponentsFromNode(node) + if (!components?.fileName) { + return + } + + if (components.isImported) { + if (!components.name) { + return + } + + return getFileSymbols(components.fileName)[components.name] + } + + const checker = getSourceFileTypeChecker(components.fileName) + + return checker.getNodeType(node) + } + + function checkForResourceInstantiations(sym: Symbol) { + const node = sym.declaration + if (!node) { + return + } + + const type = getNodeType(node) + if (type?.instantiations) { + failOnNode(`Expression instantiates resources`, node) + } + + const deps = graphCompiler.getAllDependencies(sym) + for (const d of deps) { + const node = d.declaration + if (!node) { + continue + } + + const type = getNodeType(node) + if (type?.instantiations) { + failOnNode(`Expression instantiates resources`, node) + } + } + } + + function printTypes(f: string) { + getSourceFileTypeChecker(f).printTypes() + } + + // This is basically a directive to force captured functions into + // module scope so they can be annotated. Synthesis can use this + // annotation to augment captured functions as they are synth'd. + function getMarkedNodes(node: ts.Node) { + const marked = new Set() + + function visit(node: ts.Node) { + if (!ts.isCallExpression(node)) { + return void node.forEachChild(visit) + } + + const sym = graphCompiler.getSymbol(node.expression) + if (!sym) { + return void node.forEachChild(visit) + } + + const components = getNameComponents(sym) + if (components.specifier !== '@cohesible/synapse-websites' || components.name !== 'useServer') { + return void node.forEachChild(visit) + } + + const argNode = node.arguments[0] + switch (argNode.kind) { + // These are all OK + case ts.SyntaxKind.Identifier: + case ts.SyntaxKind.ElementAccessExpression: + case ts.SyntaxKind.PropertyAccessExpression: + break + + default: + compilerError(`"useServer" is only implemented for function declarations and variables referencing functions`, argNode) + } + + const argSym = graphCompiler.getSymbol(argNode) + if (!argSym || !argSym.declaration) { + return // No point in visiting more nodes + } + + // Argument cases: + // 1. Identifier + // 1a. Function decl outside of the containing decl + // 1b. Function decl inside the containing decl + // 1c. Variable decl -> treat initializer as the argument + // 2. Arrow function / Function expression + // 3. Expression that can be reduced to one of the above (incl. call expression) + // + // For call expressions, we need to handle all possible return values. + + function getContainingDeclarationLike(n: ts.Node) { + let p = n.parent + while (p) { + // TODO: cover more cases + switch (p.kind) { + case ts.SyntaxKind.SourceFile: + case ts.SyntaxKind.ArrowFunction: + case ts.SyntaxKind.FunctionExpression: + case ts.SyntaxKind.FunctionDeclaration: + return p + } + + p = p.parent + } + + failOnNode('No container declaration found', n) + } + + const currentScope = getContainingDeclarationLike(node) + let targetScope = getContainingDeclarationLike(argSym.declaration) + while (targetScope) { + if (targetScope === currentScope) { + compilerError( + 'Target declaration must appear outside of the function scope where "useServer" is called', + annotateNode(currentScope, 'Shared scope'), + annotateNode(argSym.declaration, 'Declared here'), + annotateNode(argNode, 'Referenced here') + ) + } + targetScope = targetScope.parent + } + + if (ts.isFunctionDeclaration(argSym.declaration)) { + marked.add(argSym.declaration) + + return + } + + if (ts.isVariableDeclaration(argSym.declaration)) { + if (!argSym.declaration.initializer) return + + if (!ts.isVariableStatement(argSym.declaration.parent.parent)) return + + if (ts.isArrowFunction(argSym.declaration.initializer) || ts.isFunctionExpression(argSym.declaration.initializer)) { + marked.add(argSym.declaration) + } else { + failOnNode('Not supported with "useServer"', argSym.declaration.initializer) + } + } + } + + visit(node) + + return marked + } + + return { + registerTypes, + generateTypes, + getCallableMemberName, + getFileResourceInstantiations, + getNodeType, + checkForResourceInstantiations, + getFileSymbols, + printTypes, + hasCalledCallables, + getMarkedNodes, + } +} + +const typesFile = `[#compile]__types__.json` +async function setTypes(types: StoredTypesFilesData) { + const fs = getProgramFs() + await fs.writeJson(typesFile, sortRecord(types)) +} + +async function getTypes(fs: Pick = getProgramFs()): Promise { + return fs.readJson(typesFile).catch(throwIfNotFileNotFoundError) +} + +export async function getTypesFile(fs: Pick = getProgramFs()): Promise { + const types = await getTypes(fs) + if (!types) { + return + } + + return Object.fromEntries(Object.entries(types).map(([k, v]) => [v.outfile, v.exports])) +} \ No newline at end of file diff --git a/src/compiler/transformer.ts b/src/compiler/transformer.ts new file mode 100644 index 0000000..d3df298 --- /dev/null +++ b/src/compiler/transformer.ts @@ -0,0 +1,845 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { createObjectLiteral, getInstantiationName, failOnNode, inferName, isExported, splitExpression, isRelativeSpecifier, getNodeLocation, printNodes, createVariableStatement } from '../utils' +import { createGraphCompiler, createImporterExpression, createStaticSolver } from '../static-solver' +import type { Scope } from '../runtime/modules/core' // must be a type import! +import { parseDirectives } from '../runtime/sourceMaps' +import { getLogger } from '../logging' +import { SchemaFactory } from './validation' +import { ResourceTypeChecker } from './resourceGraph' + +// THINGS TO DO +// 1. Rewrite certain expressions/statements to work as config (must be dynamic! best to minimize rendered configuration) +// * Control flow (if, while, etc.) +// * Iterables + + +export const getFqnComponents = (sym: string) => { + const [_, module, name] = sym.match(/"(.*)"(?:\.(.*))?/) ?? [] + if (!module && !name) { + throw new Error(`Failed to get FQN of symbol: ${sym}`) + } + + if (!name) { + return { module, name: '__default' } // This is a default export! + } + + return { name, module } +} + +type FQN = `"${string}".${string}` | `"${string}"` + +function getFqn(graphCompiler: ReturnType, node: ts.Node): FQN | undefined { + const origin = node.getSourceFile()?.fileName + if (!origin) { + return + } + + const sym = graphCompiler.getSymbol(node) + if (!sym) { + return + } + + if (sym.parent && sym.parent.declaration) { + const parentFqn = getFqn(graphCompiler, sym.parent.declaration) + if (!parentFqn) { + return + } + + return `${parentFqn}.${sym.name}` as FQN + } + + if (sym.importClause) { + const moduleSpec = (sym.importClause.parent.moduleSpecifier as ts.StringLiteral).text + const resolved = isRelativeSpecifier(moduleSpec) ? path.resolve(path.dirname(origin), moduleSpec) : moduleSpec + + // Default import + const parent = sym.declaration?.parent + if (parent === sym.importClause) { + return `"${resolved.replace(/\.(t|j)sx?$/, '')}".default` + } + + if (parent && ts.isImportSpecifier(parent)) { + const name = parent.propertyName ?? parent.name + + return `"${resolved.replace(/\.(t|j)sx?$/, '')}".${name.text}` + } + + return `"${resolved.replace(/\.(t|j)sx?$/, '')}"` + } else if (sym.declaration && isExported(sym.declaration)) { + return `"${origin.replace(/\.(t|j)sx?$/, '')}".${sym.name}` + } +} + +function isExternalImport(graphCompiler: ReturnType, node: ts.Node) { + const sym = graphCompiler.getSymbol(node) + if (!sym) { + return false + } + + if (sym.parent && sym.parent.declaration) { + return isExternalImport(graphCompiler, sym.parent.declaration) + } + + if (sym.importClause) { + const moduleSpec = (sym.importClause.parent.moduleSpecifier as ts.StringLiteral).text + + return !isRelativeSpecifier(moduleSpec) // XXX: this is not entirely accurate + } + + return false +} + +export function getCoreImportName(graphCompiler: ReturnType, node: ts.Node) { + const fqn = getFqn(graphCompiler, ts.getOriginalNode(node)) + if (!fqn) { + return + } + + const { module, name } = getFqnComponents(fqn) + if (module === 'synapse:core') { + return name + } +} + +function getLibImportName(graphCompiler: ReturnType, node: ts.Node) { + const fqn = getFqn(graphCompiler, ts.getOriginalNode(node)) + if (!fqn) { + return + } + + const { module, name } = getFqnComponents(fqn) + if (module === 'synapse:lib') { + return name + } +} + +// TODO: put this cache somewhere else so it can be cleared during `watch` +const directivesCache = new Map | undefined>() +function parseHeaderDirectives(node: ts.Node) { + if (!node) { + return + } + + const sf = node.getSourceFile() + if (!sf) { + return + } + + if (directivesCache.has(node)) { + return directivesCache.get(node)! + } + + const text = sf.getFullText() + const comments = ts.getLeadingCommentRanges(text, node.pos) + if (!comments) { + directivesCache.set(node, undefined) + + return + } + + const lines = comments + .filter(c => c.kind === ts.SyntaxKind.SingleLineCommentTrivia) + .map(c => text.slice(c.pos, c.end)) + + const result = parseDirectives(lines) + directivesCache.set(node, result) + + return result +} + +export function getModuleBindingId(sourceFile: ts.SourceFile) { + const directives = parseHeaderDirectives(sourceFile) + if (!directives) { + return + } + + return directives['moduleId'] +} + +export function getTransformDirective(sourceFile: ts.SourceFile) { + const directives = parseHeaderDirectives(sourceFile) + if (!directives) { + return + } + + return directives['transform'] +} + +export function getResourceDirective(node: ts.Node) { + const directives = parseHeaderDirectives(node) + if (!directives) { + return + } + + return directives['resource'] +} + +export function getCallableDirective(node: ts.Node) { + const directives = parseHeaderDirectives(node) + if (!directives) { + return + } + + return directives['callable'] +} + +export type ResourceTransformer = ReturnType +export function createTransformer( + workingDirectory: string, // For symbols + context: ts.TransformationContext, + graphCompiler: ReturnType, + schemaFactory: SchemaFactory, + resourceTypeChecker: ResourceTypeChecker +) { + const factory = context.factory + const bindings = new Map() + const staticSolver = createStaticSolver() + const scopeSymbolProvider = createSymbolProvider() + const assignmentCache = new Map() + const needsValidationImport = new Map() + const sourceDeltas = new Map() + + let deltas: { line: number; column: number } | undefined + + function visit(node: ts.Node): ts.Node { + if (ts.isCallExpression(node)) { + const importName = getCoreImportName(graphCompiler, node.expression) + if (importName === 'addTarget' && node.arguments.length === 3) { + const [targetNode, replacementNode, targetNameNode] = node.arguments + const targetFqn = getFqn(graphCompiler, ts.getOriginalNode(targetNode)) + if (!targetFqn) { + failOnNode('No symbol found for target', targetNode) + } + + const replacementFqn = getFqn(graphCompiler, ts.getOriginalNode(replacementNode)) + if (!replacementFqn) { + // FIXME: clarify that the symbol must be exported? + failOnNode('No exported symbol found for replacement', replacementNode) + } + + const target = staticSolver.solve(targetNameNode) + if (typeof target !== 'string') { + failOnNode('Expected target name to be a string', targetNameNode) + } + + if (!bindings.has(targetFqn)) { + bindings.set(targetFqn, []) + } + + bindings.get(targetFqn)!.push({ + target, + replacement: replacementFqn, + }) + + return context.factory.createNotEmittedStatement(node) + } else if (importName === 'defineResource' || importName === 'defineDataSource') { + const n = ts.getOriginalNode(node) + const name = inferName(n) + if (!name) { + failOnNode('No name found', n) + } + + return createConfigurationClass(node, name, workingDirectory, factory) + } else if (importName === 'check') { + needsValidationImport.set(ts.getOriginalNode(node).getSourceFile(), true) + + return schemaFactory.addValidationSchema(node, factory) + } else if (importName === 'schema') { + return schemaFactory.replaceWithSchema(node, factory) + } else if (importName === 'asset') { + return factory.updateCallExpression( + node, + factory.createIdentifier('__createAsset'), + undefined, + [node.arguments[0], createImporterExpression(graphCompiler.moduleType)] + ) + } + } + + if (ts.isNewExpression(node) || ts.isCallExpression(node)) { + // if (getLibImportName(graphCompiler, node.expression) === 'Bundle' && node.arguments) { + // const sym = graphCompiler.getSymbol(node.arguments[0]) + // if (sym !== undefined) { + // resourceTypeChecker?.checkForResourceInstantiations(sym) + // } + // } + + const original = ts.getOriginalNode(node) + const bindingId = getModuleBindingId(original.getSourceFile()) + if (bindingId) { + + } else if (getCoreImportName(graphCompiler, node.expression)) { + + } else if ( + node.expression.kind !== ts.SyntaxKind.SuperKeyword && + !getCoreImportName(graphCompiler, getRootTarget(original)) && + !isInThrowStatement(node) + ) { + const p = original.parent + const isInBindFunction = ts.findAncestor(original, n => ts.isCallExpression(n) && + ['bindModel', 'bindObjectModel', 'bindFunctionModel'].includes(getCoreImportName(graphCompiler, n.expression) ?? '')) + const isDynamicImport = node.expression.kind === ts.SyntaxKind.ImportKeyword + + if (p && node.expression.kind !== ts.SyntaxKind.ParenthesizedExpression && !isInBindFunction && !isDynamicImport) { + const name = getInstantiationName(node) + if (deltas) { + const { symbol, namespace } = getSymbolForSourceMap(original, scopeSymbolProvider, deltas) + + return wrapScope( + { name, symbol, namespace }, + ts.visitEachChild(node, visit, context), + node.arguments?.some(isAsync) || isAsync(node), + context.factory + ) + } + + const { symbol, namespace } = getSymbolForSourceMap(original, scopeSymbolProvider) + const assignmentSymbol = getAssignmentSymbol(original, scopeSymbolProvider, assignmentCache) + const isNewExpression = node.kind === ts.SyntaxKind.NewExpression ? true : undefined + const ty = resourceTypeChecker.getNodeType((original as ts.CallExpression | ts.NewExpression).expression) + + return wrapScope( + { name, symbol, assignmentSymbol, namespace, isNewExpression, isStandardResource: ty?.intrinsic }, + ts.visitEachChild(node, visit, context), + node.arguments?.some(isAsync) || isAsync(node), + context.factory + ) + } + } + } + + return ts.visitEachChild(node, visit, context) + } + + function visitSourceFile(sourceFile: ts.SourceFile) { + const transformed = ts.visitEachChild(sourceFile, visit, context) + + if (needsValidationImport.get(sourceFile)) { + return ts.factory.updateSourceFile( + transformed, + [ + schemaFactory.createValidateDeclaration(), + ...sourceFile.statements, + ] + ) + } + + return transformed + } + + // We don't include the symbols for infra chunks because they contain line/column numbers + // which are directly influenced by surrounding code. This causes churn when changing a file. + // We'll have to make the positions relative to the chunk, and then somehow transform them + // back during synthesis. The easiest way is probably attaching a line delta to the chunk's metadata + function visitAsInfraChunk(node: ts.Node) { + const original = ts.getOriginalNode(node) + const sf = original.getSourceFile() + const pos = original.pos + original.getLeadingTriviaWidth(sf) + const lc = sf.getLineAndCharacterOfPosition(pos) + deltas = { line: lc.line, column: lc.character } + sourceDeltas.set(original, deltas) + const r = visit(node) + deltas = undefined + + return r + } + + return { + visit, + visitSourceFile, + visitAsInfraChunk, + bindings, + getReflectionTransformer: () => createReflectionTransformer(graphCompiler, schemaFactory), + getDeltas: (node: ts.Node) => sourceDeltas.get(node), + } +} + +function createSymbolProvider() { + const cache = new Map() + + function getIdentSymbol(node: ts.Identifier, deltas?: { line: number; column: number }) { + if (cache.has(node) && !deltas) { + return cache.get(node)! + } + + const sf = node.getSourceFile() + const pos = node.pos + node.getLeadingTriviaWidth(sf) + const lc = sf.getLineAndCharacterOfPosition(pos) + const sym: Scope['symbol'] = { + name: node.text, + line: lc.line - (deltas?.line ?? 0), + column: lc.character - (deltas?.column ?? 0), + fileName: ts.factory.createIdentifier('__filename') as any, + } + + if (!deltas) { + cache.set(node, sym) + } + + return sym + } + + return { getIdentSymbol } +} + +function isInThrowStatement(node: ts.Node) { + return !!ts.findAncestor(node, p => { + if (ts.isStatement(p)) { + return !ts.isThrowStatement(p) ? 'quit' : true + } + + return false + }) +} + +// TODO(perf): increases total compile time by around 5-10% +function getAssignmentSymbol(node: ts.Node, symbolProvider: ReturnType, cache: Map) { + if (cache.has(node)) { + return cache.get(node) + } + + const ancestor = ts.findAncestor(node, p => { + if (!p.parent) { + return false + } + + if (ts.isVariableDeclaration(p.parent) && p.parent.initializer === p) { + return true + } + + if (ts.isPropertyAssignment(p.parent) && p.parent.initializer === p) { + return true + } + + if (ts.isStatement(p.parent)) { + return 'quit' + } + + return false + }) + + cache.set(node, undefined) + + if (!ancestor) { + return + } + + const decl = ancestor.parent as ts.VariableDeclaration | ts.PropertyAssignment + if (ts.isIdentifier(decl.name)) { + const sym = symbolProvider.getIdentSymbol(decl.name) + cache.set(node, sym) + + return sym + } +} + +type TfSymbol = NonNullable +function getSymbolForSourceMap( + node: ts.Node, + symbolProvider: ReturnType, + deltas?: { line: number; column: number } +): { symbol?: TfSymbol; namespace?: TfSymbol[] } { + if (!(node as any).expression) { + return {} + } + + const expressions = splitExpression((node as ts.CallExpression | ts.NewExpression).expression).filter(ts.isIdentifier) + if (expressions.length === 0) { + return {} + } + + const lastExp = expressions.pop()! + + return { + symbol: symbolProvider.getIdentSymbol(lastExp, deltas), + namespace: expressions.length > 0 ? expressions.map(exp => symbolProvider.getIdentSymbol(exp, deltas)) : undefined, + } +} + +export interface Binding { + readonly target: string + readonly replacement: FQN +} + +function createConfigurationClass( + node: ts.CallExpression, + type: string, + workingDirectory: string, + factory = ts.factory +) { + // FIXME: this is not robust for "shared" packages. The loader should probably prepend the module specifier + const fileName = path.relative(workingDirectory, ts.getOriginalNode(node).getSourceFile().fileName) + .replace(/\.(j|t)sx?$/, '') + .replaceAll(path.sep, '--') + + getLogger().log(`resource handler ${fileName} -> "${type}"`) + + const callExp = factory.updateCallExpression( + node, + node.expression, + node.typeArguments, + [ + ...node.arguments, + factory.createStringLiteral(fileName + '--' + type + '--' + 'definition'), + ] + ) + + return callExp +} + +function isAsync(node: ts.Node): boolean { + if (ts.isAwaitExpression(node)) { + return true + } + + if ( + ts.isClassExpression(node) || + ts.isClassDeclaration(node) || + ts.isFunctionExpression(node) || + ts.isArrowFunction(node) || + ts.isFunctionDeclaration(node) || + ts.isVariableDeclaration(node) || + ts.isParameter(node) || + ts.isMethodDeclaration(node) || + ts.isGetAccessorDeclaration(node) || + ts.isSetAccessorDeclaration(node) || + ts.isConstructorDeclaration(node) + ) { + return false + } + + return node.forEachChild(n => isAsync(n)) ?? false +} + +function getRootTarget(node: ts.Node): ts.Node { + if (ts.isCallExpression(node)) { + return getRootTarget(node.expression) + } + + if (ts.isPropertyAccessExpression(node)) { + if (ts.isIdentifier(node.expression)) { + return node + } + + return getRootTarget(node.expression) + } + + if (ts.isIdentifier(node)) { + return node + } + + return node +} + +function runWithContext(args: ts.Expression[], factory = ts.factory) { + return factory.createCallExpression( + factory.createIdentifier('__scope__'), + undefined, + args + ) +} + +// foo(await bar()) +// __scope__(foo, await bar()) +// __scope__(foo, await __scope__(bar)) + +// x.y.z() +// __scope__(x.y.z.bind(x.y)) + +// new Foo(await bar()) +// __scope__(Reflect.construct.bind(Reflect, Foo, [await bar()])) +// __scope__(Reflect.construct.bind(Reflect, Foo, [await __scope__(bar)]) + +function bindNewExpression(node: ts.NewExpression) { + const reflectIdent = ts.factory.createIdentifier('Reflect') + const args = [reflectIdent, node.expression] + if (node.arguments) { + args.push(ts.factory.createArrayLiteralExpression(node.arguments)) + } + + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createPropertyAccessExpression(reflectIdent, 'construct'), + 'bind' + ), + undefined, + args, + ) +} + +function unwrapParentheses(node: ts.Expression) { + if (ts.isParenthesizedExpression(node)) { + return unwrapParentheses(node.expression) + } + + return node +} + +function bindCallExpression(node: ts.CallExpression) { + const unwrapped = unwrapParentheses(node.expression) + const thisArg = (ts.isPropertyAccessExpression(unwrapped) || ts.isElementAccessExpression(unwrapped)) + ? unwrapped.expression + : undefined + + if (!thisArg) { + return ts.factory.updateCallExpression( + node, + ts.factory.createPropertyAccessExpression( + ts.factory.createParenthesizedExpression(node.expression), + 'bind' + ), + undefined, + [ts.factory.createVoidZero(), ...node.arguments] + ) + } + + // You have to use a temp var in this case + // (_ = x.y, _.z.bind(_)) + + const tmpVarIdent = ts.factory.createIdentifier('_') + const tmpVar = ts.factory.createAssignment(tmpVarIdent, thisArg) + const targetFn = ts.isPropertyAccessExpression(unwrapped) + ? ts.factory.updatePropertyAccessExpression( + unwrapped, + tmpVarIdent, + unwrapped.name, + ) + : ts.factory.updateElementAccessExpression( + unwrapped as ts.ElementAccessExpression, + tmpVarIdent, + (unwrapped as ts.ElementAccessExpression).argumentExpression, + ) + + const bindExp = ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression(targetFn, 'bind'), + undefined, + [tmpVar, ...node.arguments] + ) + + return ts.factory.createParenthesizedExpression( + ts.factory.createCommaListExpression([tmpVar, bindExp]) + ) +} + +function wrapScope( + scope: Scope, + node: ts.NewExpression | ts.CallExpression, + isAsync = false, + factory = ts.factory +) { + const wrapped = runWithContext([ + createObjectLiteral({ ...scope } as any, factory), + factory.createArrowFunction( + isAsync ? [factory.createModifier(ts.SyntaxKind.AsyncKeyword)] : undefined, + undefined, + [], + undefined, + undefined, + node + ) + ], factory) + + return isAsync ? factory.createAwaitExpression(wrapped) : wrapped +} + +export function hasTraceDirective(node: ts.Node) { + const sf = ts.getOriginalNode(node).getSourceFile() + if (!sf) { + return + } + + const text = sf.getFullText(sf) + const comments = ts.getLeadingCommentRanges(text, node.pos) + if (!comments) { + return + } + + const lines = comments + .filter(c => c.kind === ts.SyntaxKind.SingleLineCommentTrivia) + .map(c => text.slice(c.pos, c.end)) + + const result = parseDirectives(lines) + + return result['trace'] +} + +function createStubFactory(moduleName: string) { + function createFunc(block: ts.Block, params: ts.ParameterDeclaration[] = []) { + return ts.factory.createFunctionExpression( + undefined, + undefined, + undefined, + undefined, + params, + undefined, + block + ) + } + + function createThrow(message: string) { + return ts.factory.createThrowStatement( + ts.factory.createNewExpression( + ts.factory.createIdentifier('Error'), + undefined, + [ts.factory.createStringLiteral(message)] + ) + ) + } + + // This enables the (rough) equivalent of an "allow-undefined-symbols" option available in most linkers + // Of course this operates more so at the library level rather than individual symbols + // + // We only need to add this because the synth loader checks for this property before serializing. + const moduleIdOverride = ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Symbol'), + 'for' + ), + undefined, + [ts.factory.createStringLiteral('moduleIdOverride')] + ) + + const get = createFunc(ts.factory.createBlock([ + ts.factory.createIfStatement( + ts.factory.createStrictEquality(ts.factory.createIdentifier('prop'), moduleIdOverride), + ts.factory.createReturnStatement() + ) + ]), [ + ts.factory.createParameterDeclaration(undefined, undefined, '_'), + ts.factory.createParameterDeclaration(undefined, undefined, 'prop') + ]) + + const errorMessage = `Module "${moduleName}" has not been deployed` + const proxyFunctions: { [P in keyof ProxyHandler]: ts.FunctionExpression | ts.ArrowFunction } = { + get, + set: createFunc(ts.factory.createBlock([createThrow(errorMessage)])), + apply: createFunc(ts.factory.createBlock([createThrow(errorMessage)])), + construct: createFunc(ts.factory.createBlock([createThrow(errorMessage)])), + } + + const body = ts.factory.createBlock([ + ts.factory.createReturnStatement( + ts.factory.createNewExpression( + ts.factory.createIdentifier('Proxy'), + undefined, + [ + createFunc(ts.factory.createBlock([])), + createObjectLiteral(proxyFunctions) + ] + ) + ) + ]) + + return ts.factory.createFunctionDeclaration(undefined, undefined, 'createStub', undefined, [], undefined, body) +} + +export function generateModuleStub(tscRootDir: string, graphCompiler: ReturnType, sf: ts.SourceFile) { + const exports: string[] = [] + for (const s of sf.statements) { + if (!isExported(s)) { + continue + } + + if (ts.isVariableStatement(s)) { + for (const d of s.declarationList.declarations) { + const sym = graphCompiler.getSymbol(d) + if (sym) { + exports.push(sym.name) + } + } + } else { + const sym = graphCompiler.getSymbol(s) + if (sym) { + exports.push(sym.name) + } + } + } + + const moduleName = path.relative(tscRootDir, sf.fileName) + const stubFactory = createStubFactory(moduleName) + const statements: ts.Statement[] = [] + for (const name of exports) { + statements.push( + createVariableStatement( + name, + ts.factory.createCallExpression(stubFactory.name!, undefined, undefined), + [ts.SyntaxKind.ExportKeyword] + ), + ) + } + + return ts.factory.updateSourceFile( + sf, + [stubFactory, ...statements], + ) +} + +function createReflectionTransformer( + graphCompiler: ReturnType, + schemaFactory: SchemaFactory +): ts.PrintHandlers { + function onEmitNode(hint: ts.EmitHint, node: ts.Node, emitCallback: (hint: ts.EmitHint, node: ts.Node) => void) { + if (!ts.isCallExpression(node)) { + return emitCallback(hint, node) + } + + const importName = getCoreImportName(graphCompiler, node.expression) + if (importName) { + console.log(getNodeLocation(node), importName) + } + + if (importName === 'check') { + // needsValidationImport.set(ts.getOriginalNode(node).getSourceFile(), true) + + return schemaFactory.addValidationSchema(node) + } else if (importName === 'schema') { + return ts.factory.createCallExpression( + ts.factory.createIdentifier('foo'), + undefined, [] + ) + // return schemaFactory.replaceWithSchema(node) + } + + return emitCallback(hint, node) + } + + function isEmitNotificationEnabled(node: ts.Node) { + return node.kind === ts.SyntaxKind.CallExpression + } + + function substituteNode(hint: ts.EmitHint, node: ts.Node): ts.Node { + if (!ts.isCallExpression(node)) { + return node + } + + const importName = getCoreImportName(graphCompiler, node.expression) + if (importName === 'check') { + // needsValidationImport.set(ts.getOriginalNode(node).getSourceFile(), true) + + return schemaFactory.addValidationSchema(node) + } else if (importName === 'schema') { + // Updating the call expression is a hack. Emitting the literal directly didn't work + return ts.factory.updateCallExpression( + node, + ts.factory.createPropertyAccessExpression( + ts.factory.createCallExpression( + ts.factory.createIdentifier('require'), + undefined, + [ts.factory.createStringLiteral('')], // TODO + ), + '__schema' + ), + undefined, + [schemaFactory.replaceWithSchema(node)] + ) + } + + return node + } + + return { + // onEmitNode + substituteNode, + } +} \ No newline at end of file diff --git a/src/compiler/validation.ts b/src/compiler/validation.ts new file mode 100644 index 0000000..56c8cc5 --- /dev/null +++ b/src/compiler/validation.ts @@ -0,0 +1,381 @@ +import ts from 'typescript' +import type { Schema } from '../runtime/modules/validation' +import { createObjectLiteral, createVariableStatement, failOnNode, printNodes } from '../utils' + +function getOriginalNode(node: T): T { + node = ts.getOriginalNode(node) as T + const sf = node.getSourceFile() + const resolved = (sf as any).getOriginalSourceFile?.() ?? sf + if (resolved === sf) { + return node + } + + function findNodeByPos(target: ts.Node, pos: number) { + function visit(node: ts.Node): ts.Node | undefined { + if (node.pos === pos) { + return node + } + + return node.forEachChild(visit) + } + + const found = visit(target) + if (!found) { + failOnNode('Unable to find original node', node) + } else if (found.kind !== node.kind) { + failOnNode('Found node is not the same kind as the target', found) + } + + return found + } + + return findNodeByPos(resolved, node.pos) as T +} + +export type SchemaFactory = ReturnType +export function createSchemaFactory(program: ts.Program) { + const validateIdent = ts.factory.createIdentifier('validate') + + function createValidateDeclaration() { + const runtime = ts.factory.createCallExpression( + ts.factory.createIdentifier('require'), + undefined, + [ts.factory.createStringLiteral('synapse:validation')] + ) + + const validateFn = ts.factory.createPropertyAccessExpression(runtime, validateIdent) + const validateDecl = createVariableStatement(validateIdent, validateFn) + + return validateDecl + } + + function createSchema(typeNode: ts.TypeNode): Schema { + return typeToSchema(program.getTypeChecker(), typeNode) + } + + function addValidationSchema(node: ts.CallExpression, factory = ts.factory) { + node = getOriginalNode(node) + + const type = node.typeArguments?.[0] + if (!type) { + failOnNode('No type annotation exists', node) + } + + const schema = createSchema(type) + + return factory.updateCallExpression( + node, + validateIdent, + undefined, + [node.arguments[0], createObjectLiteral(schema as any, factory)] + ) + } + + function replaceWithSchema(node: ts.CallExpression, factory = ts.factory) { + node = getOriginalNode(node) + + const type = node.typeArguments?.[0] + if (!type) { + failOnNode('No type annotation exists', node) + } + + const schema = createSchema(type) + + return createObjectLiteral(schema as any, factory) + } + + return { + createValidateDeclaration, + createSchema, + addValidationSchema, + replaceWithSchema, + } +} + +function typeToSchema(typeChecker: ts.TypeChecker, typeOrNode: ts.Type | ts.TypeNode): Schema { + const node = (typeOrNode as any).kind ? typeOrNode as ts.TypeNode : undefined + const type = node ? typeChecker.getTypeFromTypeNode(node) : typeOrNode as ts.Type + + // Special case: `boolean` is evaluated as `true | false` but we want just `boolean` + if (type.flags & ts.TypeFlags.Boolean) { + return { type: 'boolean' } + } + + if (type.isUnion()) { + const types = type.types + const enums: any[] = [] + const schemas: Schema[] = [] + for (const t of types) { + const subschema = typeToSchema(typeChecker, t) + if (subschema.const) { + enums.push(subschema.const) + } else if (subschema.enum) { + enums.push(...subschema.enum) + } else { + schemas.push(subschema) + } + } + + // Collapse it down to a boolean + if (enums.includes(true) && enums.includes(false)) { + schemas.push({ type: 'boolean' }) + + const filtered = enums.filter(x => typeof x !== 'boolean') + const enumSchemas = filtered.length > 0 ? [{ enum: Array.from(new Set(filtered)) }] : [] + + return { anyOf: [...enumSchemas, ...schemas] } as Schema + } + + const enumSchemas = enums.length > 0 ? [{ enum: Array.from(new Set(enums)) }] : [] + + return { anyOf: [...enumSchemas, ...schemas] } as Schema + } + + if (type.isIntersection()) { + function intersect(a: Schema, b: Schema): Schema | false { + if (b.anyOf) { + const schemas = b.anyOf.map(s => intersect(a, { ...b, anyOf: undefined, ...s })).filter((s): s is Schema => !!s) + if (schemas.length === 0) { + return false + } + + return { anyOf: schemas } as any + } + + if (a.anyOf) { + return intersect(b, a) + } + + if (a.enum && b.enum) { + const union = new Set([...a.enum, ...b.enum]) + const intersection = Array.from(union).filter(x => a.enum!.includes(x) && b.enum!.includes(x)) + if (intersection.length === 0) { + return false + } + + return { enum: intersection } as Schema + } + + if (a.enum && b.type) { + const enums: any[] = [] + if (Array.isArray(b.type)) { + const schemas = b.type.map(s => intersect(a, { ...b, type: s })).filter((s): s is Schema => !!s) + for (const s of schemas) { + if (s.enum) { + enums.push(...s.enum) + } + } + } else { + for (const x of a.enum) { + switch (b.type) { + case 'boolean': + case 'string': + case 'number': + if (typeof x === b.type) { + enums.push(x) + } + + break + case 'null': + if (x === null) { + enums.push(x) + } + + break + } + } + } + + if (enums.length === 0) { + return false + } + + return { enum: Array.from(new Set(enums)) } as Schema + } + + if (a.type && b.enum) { + return intersect(b, a) + } + + if (a.type && b.type) { + if (Array.isArray(b.type)) { + const schemas = b.type.map(s => intersect(a, { ...b, type: s })).filter((s): s is Schema => !!s) + + return { type: schemas.map(s => s.type) } as any + } + if (Array.isArray(a.type)) { + return intersect(b, a) + } + + if (a.type !== b.type) { + return false + } + + if (a.type === 'object' && b.type === 'object') { + const keysA = Object.keys(a.properties ?? {}) + const keysB = Object.keys(b.properties ?? {}) + const keys = Array.from(new Set([...keysA, ...keysB])) + const props: Record = {} + + for (const key of keys) { + const schemaA = a.properties?.[key] + const schemaB = b.properties?.[key] + if (schemaA && schemaB) { + const s = intersect(schemaA, schemaB) + if (s) { + props[key] = s + } + } else if (schemaA) { + props[key] = schemaA + } else if (schemaB) { + props[key] = schemaB + } + } + + const requiredA = a.required ?? [] + const requiredB = b.required ?? [] + const required = Array.from(new Set([...requiredA, ...requiredB])) + + // `never` is effectively coerced to `any` here + return { type: 'object', properties: props, required } + } + + if (a.type === 'array' && b.type === 'array') { + const items = a.items && b.items ? intersect(a.items, b.items) : a.items ?? b.items + + if (a.prefixItems && b.prefixItems) { + if (a.prefixItems.length !== b.prefixItems.length) { + return false + } + + const prefixItems: Schema[] = [] + for (let i = 0; i < a.prefixItems.length; i++) { + const s = intersect(a.prefixItems[i], b.prefixItems[i]) + if (!s) { + return false + } + prefixItems.push(s) + } + + if (items) { + return { type: 'array', items, prefixItems } + } + + return { type: 'array', prefixItems } + } + + if (!items) { + return false + } + + return { type: 'array', items } + } + + return { type: a.type } + } + + return false + } + + const res = type.types.map(t => typeToSchema(typeChecker, t)).reduce((a, b) => a && b ? intersect(a, b) : false) + if (!res) { + throw new Error('Failed to create intersection schema') + } + + return res + } + + const typeAsString = typeChecker.typeToString(type) + if (typeAsString === 'any') { + return { type: 'object' } // FIXME: not correct + } else if (typeAsString === 'string') { + return { type: 'string' } + } else if (typeAsString === 'number') { + return { type: 'number' } + } else if (typeAsString === 'true' || typeAsString === 'false') { + return { type: 'boolean', enum: [typeAsString === 'true' ? true : false] } + } else if (type.isNumberLiteral() || type.isStringLiteral()) { + return { + type: type.isNumberLiteral() ? 'number' : 'string', + enum: [type.value] + } + } + + const symbol = type.symbol ?? (node ? typeChecker.getSymbolAtLocation(node) : undefined) + const typeNode = node ?? typeChecker.typeToTypeNode(type, symbol?.valueDeclaration, undefined) + if (typeNode === undefined) { + throw new Error(`Type "${typeAsString}" has no type node`) + } + + function convertSymbols(symbols: ts.Symbol[]) { + const properties: Record = {} + const required: string[] = [] + + for (const prop of symbols) { + if (!prop.valueDeclaration) { + throw new Error(`Symbol "${prop.name}" does not have a value declaration`) + } + const propType = typeChecker.getTypeOfSymbolAtLocation(prop, prop.valueDeclaration) + const nonNullableType = propType.getNonNullableType() + properties[prop.name] = typeToSchema(typeChecker, nonNullableType) + + if (nonNullableType === propType) { + required.push(prop.name) + } + } + + if (required.length === 0) { + return { + type: 'object' as const, + properties, + } + } + + return { + type: 'object' as const, + properties, + required, + } + } + + if (type.isClassOrInterface() || ts.isTypeLiteralNode(typeNode)) { + return convertSymbols(typeChecker.getPropertiesOfType(type)) + } else if (ts.isTypeReferenceNode(typeNode)) { + const args = typeChecker.getTypeArguments(type as ts.TypeReference) + + if (ts.isIdentifier(typeNode.typeName) && args) { + if (typeNode.typeName.text === 'Record') { + return { + type: 'object', + additionalProperties: typeToSchema(typeChecker, args[1]), + // key: typeToSchema(typeChecker, args[0]), + // value: typeToSchema(typeChecker, args[1]), + } + } else if (typeNode.typeName.text === 'Array') { + return { + type: 'array', + items: typeToSchema(typeChecker, args[0]) + } + } else if (typeNode.typeName.text === 'Promise') { + // Assume return type position + return typeToSchema(typeChecker, args[0]) + } + } + } else if (ts.isArrayTypeNode(typeNode)) { + const args = typeChecker.getTypeArguments(type as ts.TypeReference) + + return { + type: 'array', + items: typeToSchema(typeChecker, args[0]), + } + } else if (ts.isTupleTypeNode(typeNode)) { + return { + type: 'array', + prefixItems: typeNode.elements.map(item => typeToSchema(typeChecker, item)), + } + } else if (symbol.members) { + return convertSymbols([...symbol.members.values()]) + } + + throw new Error(`Unable to convert type "${typeAsString}" to JSON schema`) +} \ No newline at end of file diff --git a/src/deploy/deployment.ts b/src/deploy/deployment.ts new file mode 100644 index 0000000..a35e093 --- /dev/null +++ b/src/deploy/deployment.ts @@ -0,0 +1,1467 @@ +import * as path from 'node:path' +import * as fs from 'node:fs/promises' +import * as child_process from 'node:child_process' +import { DeploymentContext, startService } from './server' +import { BuildTarget, LocalWorkspace, getProviderCacheDir, getBuildDir } from '../workspaces' +import type { TfJson } from '../runtime/modules/terraform' +import EventEmitter from 'node:events' +import { Logger, OutputContext, getLogger, runTask } from '../logging' +import { TemplateService } from '../templates' +import { isErrorLike, isWindows, keyedMemoize, memoize, sortRecord, strcmp } from '../utils' +import { LoadedBackendClient } from '../backendClient' +import { getFs } from '../execution' +import { TfState } from './state' +import { randomUUID } from 'node:crypto' +import { getFsFromHash, getDeploymentFs, getProgramFs, getProgramHash, getResourceProgramHashes, getTemplate, putState, setResourceProgramHashes } from '../artifacts' +import { runCommand } from '../utils/process' +import { readKey } from '../cli/config' +import { getDisplay, spinners } from '../cli/ui' +import { readDirRecursive } from '../system' + +export interface DeployOptions { + serverPort?: number + workingDirectory?: string + consoleLogger?: boolean | CustomLogger + updateProviders?: boolean + autoApprove?: boolean + targetResources?: string[] + targetFiles?: string[] + replaceResource?: string | string[] + parallelism?: number + disableRefresh?: boolean + terraformPath?: string + useTests?: boolean + target?: string + sharedLib?: boolean + + // Persists a session for repeat deployments + keepAlive?: boolean + + cpuProfile?: boolean + + skipCommit?: boolean + + syncAfter?: boolean // For CI + noSave?: boolean + + // XXX + /** @deprecated */ + workspaceConfig?: LocalWorkspace + stateLocation?: string +} + +type CustomLogger = (entry: TfJsonOutput) => void + +export type DeploymentStatus = 'refreshing' | 'pending' | 'applying' | 'complete' | 'failed' | 'waiting' + + +export function mapResource(r: TfState['resources'][number] | undefined): typeof r { + if (r && 'instances' in r) { + return { + ...r, + state: (r as any).instances[0], + } + } + + return r +} + +export function getSynapseResourceType(r: TfState['resources'][number] | undefined): string | undefined { + if (r?.type !== 'synapse_resource') return + + return mapResource(r)?.state?.attributes.type +} + +export function getSynapseResourceInput(r: TfState['resources'][number]): any { + return mapResource(r)?.state.attributes.input.value +} + +export function getSynapseResourceOutput(r: TfState['resources'][number]): any { + return mapResource(r)?.state.attributes.output.value +} + +function maybeExtractError(takeError: (requestId: string) => unknown | undefined, reason?: string) { + const requestId = reason?.match(/x-synapse-request-id: ([\w]+)/)?.[1] // TODO: only need to check for this w/ custom resources + if (!requestId) { + return + } + + const maybeError = takeError(requestId) + if (isErrorLike(maybeError)) { + return maybeError + } +} + +function createInitView() { + const view = getDisplay().getOverlayedView() + const getRow = keyedMemoize((addr: string) => view.createRow(undefined, undefined, spinners.braille)) + + function getProgressText(ev: InstallProviderEvent) { + switch (ev.phase) { + case 'downloading': { + const downloaded = ev.downloaded ?? 0 + const size = ev.size + const percentage = size !== undefined ? Math.floor((downloaded / size) * 100) : undefined + + return percentage !== undefined ? `${ev.phase} (${percentage}%)` : ev.phase + } + + } + + return ev.phase + } + + function handleEvent(ev: InstallProviderEvent) { + if (ev.phase === 'complete' || ev.error) { + return getRow(ev.address).destroy() + } + + const label = `${ev.name}-${ev.version}` + + getRow(ev.address).update(`${label}: ${getProgressText(ev)}`) + } + + function dispose() { + + } + + return { handleEvent, dispose } +} + +export async function createZip(files: Record, dest: string) { + const tfPath = await getTerraformPath() + const req: string[] = [dest, ...Object.entries(files).sort((a, b) => strcmp(a[0], b[0]))].flat() + await runCommand(tfPath, ['zip'], { input: req.join('\n') }) +} + +export async function createZipFromDir(dir: string, dest: string, includeDir = false) { + const tfPath = await getTerraformPath() + const files = await readDirRecursive(getFs(), dir) + if (includeDir) { + const base = path.basename(dir) + const filesWithDir: Record = {} + for (const [k, v] of Object.entries(files)) { + filesWithDir[path.join(base, k)] = v + } + const req: string[] = [dest, ...Object.entries(filesWithDir).sort((a, b) => strcmp(a[0], b[0]))].flat() + await runCommand(tfPath, ['zip'], { input: req.join('\n') }) + } else { + const req: string[] = [dest, ...Object.entries(files).sort((a, b) => strcmp(a[0], b[0]))].flat() + await runCommand(tfPath, ['zip'], { input: req.join('\n') }) + } +} + +function createTerraformLogger( + takeError?: (requestId: string) => unknown | undefined, + logger: Pick = getLogger(), + onDiagnostic?: (diag: Error | TfDiagnostic) => void, +): CustomLogger { + // We don't want to bubble up diagnostics that are handled somewhere else + const handledDiags = new Set() // This is a memory leak + + const getInitView = memoize(createInitView) + + return function (entry) { + switch (entry.type) { + case 'change_summary': { + logger.debug(`Change summary:`, entry.changes) + if (entry.changes.operation === 'apply') { + logger.emitDeploySummaryEvent(entry.changes) + } + break + } + case 'resource_drift': { + logger.debug(`Resource drift:`, entry.change.resource.addr) + break + } + case 'planned_change': { + const resource = entry.change.resource + logger.debug(`Planned change (${entry.change.action}): ${resource.resource}`) + + // TODO: remove `move` ? + if (entry.change.action !== 'move') { + logger.emitDeployEvent({ + action: entry.change.action, + resource: resource.resource, + status: 'pending', + }) + } + + break + } + case 'apply_start': { + const resource = entry.hook.resource + // TODO: track how much time Terraform is spending w/ serialized data + + logger.debug(`Applying configuration (${entry.hook.action}): ${resource.resource}`) + + logger.emitDeployEvent({ + action: entry.hook.action, + resource: resource.resource, + status: 'applying', + }) + + break + } + case 'apply_progress': { + const resource = entry.hook.resource + logger.debug(`Still applying configuration (${entry.hook.action}) [${entry.hook.elapsed_seconds}s elapsed]: ${resource.resource}`) + + break + } + case 'apply_complete': { + const resource = entry.hook.resource + // TODO: track how much time Terraform is spending w/ serialized data + + logger.debug(`Finished applying: ${resource.resource}`) + + logger.emitDeployEvent({ + action: entry.hook.action, + resource: resource.resource, + status: 'complete', + state: entry.hook.state, + }) + + break + } + case 'apply_errored': { + const resource = entry.hook.resource + logger.debug(`Failed to ${entry.hook.action}: ${resource.resource}`) + + const reason = entry.hook.reason + + if (takeError) { + const maybeError = maybeExtractError(takeError, reason) + if (maybeError) { + handledDiags.add(reason!) + onDiagnostic?.(maybeError) + + return logger.emitDeployEvent({ + action: entry.hook.action, + resource: resource.resource, + status: 'failed', + reason: maybeError as Error, + } as any) + } + } + + logger.emitDeployEvent({ + action: entry.hook.action, + resource: resource.resource, + status: 'failed', + reason: entry.hook.reason, + } as any) + + break + } + case 'refresh_start': { + const resource = entry.hook.resource + logger.debug(`Refreshing: ${resource.resource}`) + + logger.emitDeployEvent({ + action: 'read', + resource: resource.resource, + status: 'refreshing', + }) + + break + } + case 'refresh_complete': { + const resource = entry.hook.resource + logger.debug(`Finished refreshing: ${resource.resource}`) + + logger.emitDeployEvent({ + action: 'read', + resource: resource.resource, + status: 'complete', + }) + + break + } + case 'diagnostic': { + const diags = entry.diagnostic ? [entry.diagnostic] : entry.diagnostics! + for (const diag of diags) { + if (onDiagnostic) { + if (!handledDiags.has(diag.summary)) { + const err = takeError ? maybeExtractError(takeError, diag.summary) : undefined + onDiagnostic(err ?? diag) + } + } else { + logger.debug('Diagnostics:', diag.summary) + // logger.debug('Diagnostics (highlight):', getHighlightFromDiagnostic(diag), diag.range?.start, diag.range?.end) + + if (diag.detail) { + logger.debug('Diagnostics (detail):', diag.detail) + } + } + } + + break + } + case 'error': + logger.error(`TF error:`, entry.data) + break + case 'plan': + const plan = parsePlan(entry.data) + logger.emitPlanEvent({ plan }) + break + case 'install_provider': + getInitView().handleEvent(entry.hook) + break + default: + if ((entry as any).type === 'version') { + logger.debug(`Running terraform (version: ${(entry as any).terraform})`) + } else { + logger.debug('Unknown', JSON.stringify(entry, undefined, 4)) + } + } + } +} + +async function listProviders(dir: string) { + const result: { source: string, name: string, version: string }[] = [] + + try { + for (const f of await fs.readdir(dir, { withFileTypes: true })) { + if (f.isDirectory() || f.isSymbolicLink()) { + const source = f.name + const sourcePath = path.resolve(dir, source) + const organizations = await fs.readdir(sourcePath, { withFileTypes: true }) + for (const o of organizations) { + if (!f.isDirectory() && !f.isSymbolicLink()) continue + + const orgName = o.name + const orgPath = path.resolve(sourcePath, orgName) + const providers = await fs.readdir(orgPath, { withFileTypes: true }) + for (const p of providers) { + if (!f.isDirectory() && !f.isSymbolicLink()) continue + + const name = `${orgName}/${p.name}` + const providerPath = path.resolve(orgPath, p.name) + const versions = await fs.readdir(providerPath, { withFileTypes: true }) + for (const v of versions) { + result.push({ source, name, version: v.name }) + + // you can go 1 more layer deep to find the os/arch + } + } + } + } + } + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + + return + } + + return result +} + +async function lockFileExists(dir: string) { + try { + await fs.readFile(path.resolve(dir, '.terraform.lock.hcl')) + return true + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + return false + } +} + +interface ResourceAddress { + readonly type: string + readonly name: string +} + +interface FindMovesResult { + readonly score: number + readonly moves: { from: ResourceAddress; to: ResourceAddress }[] +} + +export type TerraformSession = Awaited> + +interface DeployResult { + readonly error?: Error + readonly state: TfState +} + +// [{"subject":"data.synapse_resource.bar","{"type":"property","value":"foo"}]}],"type":"result"} +//expressions":[{"type":"property","value":"output"}, + +type TfExp = { type: 'property', value: string } | { type: 'element', value: number } +export interface TfRef { + readonly subject: string + readonly expressions: TfExp[] +} + +export interface BoundTerraformSession { + plan: (opt?: DeployOptions) => Promise + apply: (opt?: DeployOptions) => Promise + destroy: (opt?: DeployOptions) => Promise + getState: () => Promise + getRefs: (targets: string[]) => Promise> + setTemplate: (template: TfJson) => Promise // XXX: this is kind of a hack + dispose: () => Promise + readonly templateService: TemplateService + readonly moduleLoader: ReturnType +} + +export interface SessionContext extends DeploymentContext { + readonly buildTarget: BuildTarget & { programHash?: string } + readonly templateService: TemplateService + readonly backendClient: LoadedBackendClient + readonly terraformPath: string +} + +class TfError extends Error { + public constructor(summary: string, public readonly detail?: string, public readonly range?: TfDiagnostic['range']) { + super(summary) + + // if (range) { + // getSnippet(this).then(m => process.stderr.write(`${m}\n`)) + // } + } +} + +async function getSnippet(err: TfError, context = 25) { + if (!err.range) return + + const rawTemplate = await getProgramFs().readFile('template.json', 'utf-8') + const source = rawTemplate.slice(err.range.start.column - 1 - context, err.range.end.column - 1 + context) + + return source +} + +export class SessionError extends AggregateError {} + +// {"@level":"error","@message":"Error: Inconsistent dependency lock file","@module":"terraform.ui","@timestamp":"2024-03-29T16:55:58.216044-07:00","diagnostic":{"severity":"error","summary":"Inconsistent dependency lock file","detail":"The following dependency selections recorded in the lock file are inconsistent with the current configuration:\n - provider registry.terraform.io/hashicorp/aws: required by this configuration but no version is selected\n\nTo make the initial dependency selections that will initialize the dependency lock file, run:\n terraform init"},"type":"diagnostic"} + +export async function startTerraformSession( + context: SessionContext, + args: string[] = [], + opt?: DeployOptions, +) { + const diags: (Error | TfDiagnostic)[] = [] + const templateFile = await context.templateService.getTemplateFilePath() + const templateDir = path.dirname(templateFile) + const server = await startService(context, context.buildTarget.workingDirectory, opt?.serverPort, templateDir) + const logger = createTerraformLogger(server.takeError, undefined, d => diags.push(d)) + + function mapDiag(d: Error | TfDiagnostic) { + if (isErrorLike(d)) { + return d + } + + return new TfError(d.summary, d.detail, d.range) + } + + function checkDiags() { + if (diags.length === 0) { + return + } + + if (diags.length === 1) { + const err = mapDiag(diags[0]) + diags.length = 0 + throw err + } + + const err = new SessionError(diags!.map(mapDiag), `Operation failed`) + diags.length = 0 + throw err + } + + const dataDir = path.resolve(templateDir, 'data') + await getFs().link(context.dataDir, dataDir, { + symbolic: true, + typeHint: isWindows() ? 'junction' : 'dir', + overwrite: false, + }) + + const env = { + ...process.env, + TF_SYNAPSE_PROVIDER_ENDPOINT: `http://localhost:${server.port}`, + TF_SYNAPSE_PROVIDER_WORKING_DIRECTORY: context.buildTarget.workingDirectory, + // TF_SYNAPSE_PROVIDER_OUTPUT_DIR: '', + // TF_SYNAPSE_PROVIDER_BUILD_DIR: '', + + TF_PLUGIN_CACHE_DIR: getProviderCacheDir(), + TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE: '1', + + TF_AWS_GLOBAL_TIME_MODIFIER: '0.25', + // TF_LOG: 'TRACE', + } + + const resolvedArgs = ['start-session', ...args, '-json'] + + if (opt?.cpuProfile) { + resolvedArgs.push('-cpu-profile', `terraform.prof`) + } + + const c = child_process.spawn(context.terraformPath, resolvedArgs, { + cwd: path.dirname(templateFile), + stdio: 'pipe', + env, + }) + + c.on('exit', (code, signal) => { + server.dispose() + stateEmitter.emit('ready', new Error(`Failed to start session (exit code: ${code}): ${buffer}`)) + }) + + let isReady = false + const stateEmitter = new EventEmitter() + + function waitForReady() { + if (isReady) { + return Promise.resolve() + } + + return new Promise((resolve, reject) => { + stateEmitter.once('ready', (err?: Error) => err ? reject(err) : resolve()) + }).finally(checkDiags) + } + + function waitForResult() { + return new Promise((resolve, reject) => { + stateEmitter.once('result', (data?: any, err?: Error) => err ? reject(err) : resolve(data)) + }).finally(checkDiags) + } + + function waitForPlan(): Promise> { + return new Promise((resolve, reject) => { + // TODO: listen for `error` events + stateEmitter.once('plan', (data?: any, err?: Error) => err ? reject(err) : resolve(data)) + }) + } + + let buffer = '' + function onData(d: Buffer) { + const lines = (buffer + d).split('\n') + buffer = lines.pop() ?? '' + + for (const l of lines) { + try { + const msg = JSON.parse(l) as TfJsonOutput | { type: 'ready' } // TODO: remove `ready` and just use `result` + + if (msg.type === 'ready') { + isReady = true + stateEmitter.emit('ready') + stateEmitter.emit('result', undefined, undefined) + } else if (msg.type === 'result') { + stateEmitter.emit('result', msg.data) + } else { + logger(msg) + if (msg.type === 'plan') { + stateEmitter.emit('plan', msg.data) + } else if (msg.type === 'diagnostic') { + // if (msg.diagnostic) { + // const err = new AxonError(msg.diagnostic.summary, msg.diagnostic.detail) + // stateEmitter.emit('result', undefined, err) + // } else { + // const agg = new AggregateError( + // msg.diagnostics!.map(diag => new AxonError(diag.summary, diag.detail)), + // `Deployment error` + // ) + // stateEmitter.emit('result', undefined, agg) + // } + } + } + } catch(e) { + getLogger().debug('Bad parse', e) + // Failure is due to non-JSON data + } + } + } + + // TODO: try to kill process on cancel + + c.stdout.on('data', onData) + c.stderr.on('data', d => getLogger().raw(d)) + + function write(chunk: string | Buffer) { + return new Promise((resolve, reject) => { + c.stdin.write(chunk, err => err ? reject(err) : resolve()) + }) + } + + await waitForReady() + + const init = async () => { + isReady = false + await write(`${['init'].join(' ')}\n`) + + return waitForReady() + } + + async function needsInit() { + const tfDir = path.dirname(templateFile) + const cacheDir = getProviderCacheDir() + const stateFile = path.resolve(tfDir, '.terraform', 'terraform.tfstate') + + const [hasPluginCache, hasLockFile, hasStateFile] = await Promise.all([ + getFs().fileExists(cacheDir), + lockFileExists(tfDir), + getFs().fileExists(stateFile), + ]) + + if (!hasPluginCache) { + await fs.mkdir(cacheDir, { recursive: true }) + if (hasStateFile) { + getLogger().warn(`Provider cache is missing but state file exists. Deleting previously installed providers.`) + await getFs().deleteFile(path.dirname(stateFile)) + } + + return true + } + + return !hasLockFile || !hasStateFile + } + + await runTask('init', 'terraform', async () => { + if (await needsInit()) { + getLogger().log(`Initializing providers`) + await init() + } + }, 10) + + return { + apply: async (opt?: DeployOptions) => { + isReady = false + await write(`${['apply', ...getDeployOptionsArgs(opt)].join(' ')}\n`) + + return waitForResult() + }, + destroy: async (opt?: DeployOptions) => { + isReady = false + await write(`${['apply', '-destroy', ...getDeployOptionsArgs(opt)].join(' ')}\n`) + + return waitForReady() + }, + plan: async (opt?: DeployOptions) => { + isReady = false + const result = waitForPlan() + await write(`${['plan', ...getDeployOptionsArgs(opt)].join(' ')}\n`) + await waitForReady() + + return result + }, + reloadConfig: async () => { + isReady = false + await write(`${['reload-config'].join(' ')}\n`) + + return waitForReady() + }, + getState: async () => { + isReady = false + const result = waitForResult() + await write(`${['get-state'].join(' ')}\n`) + await waitForReady() + + return result + }, + setState: async (fileName: string) => { + isReady = false + await write(`${['set-state', fileName].join(' ')}\n`) + + return waitForReady() + }, + getRefs: async (targets: string[]) => { + isReady = false + const result = waitForResult>() + await write(`${['get-refs', ...targets].join(' ')}\n`) + await waitForReady() + + return result + }, + findMoves: async (oldTemplate: string) => { + isReady = false + const result = waitForResult() + await write(`${['find-moves', oldTemplate].join(' ')}\n`) + await waitForReady() + + return result + }, + dispose: () => write('exit\n'), + } +} + +export async function startStatelessTerraformSession( + template: TfJson, + terraformPath: string +) { + // const diags: (Error | TfDiagnostic)[] = [] + const logger = createTerraformLogger() + + const templateFilePath = path.resolve(getBuildDir(), 'tmp', 'stack.tf.json') + await getFs().writeFile(templateFilePath, JSON.stringify(template)) + + function checkDiags() { + // if (diags.length === 0) { + // return + // } + + // if (diags.length === 1) { + // throw new AxonError(diags[0].summary, diags[0].detail) + // } + + // throw new AggregateError( + // diags!.map(diag => new AxonError(diag.summary, diag.detail)), + // `Deployment error` + // ) + } + + const c = child_process.spawn(terraformPath, ['start-session', '-json'], { + cwd: path.dirname(templateFilePath), + stdio: 'pipe', + }) + + c.on('exit', (code, signal) => { + stateEmitter.emit('ready', new Error(`Failed to start session (exit code: ${code}): ${buffer}`)) + }) + + let isReady = false + const stateEmitter = new EventEmitter() + + function waitForReady() { + if (isReady) { + return Promise.resolve() + } + + return new Promise((resolve, reject) => { + stateEmitter.once('ready', (err?: Error) => err ? reject(err) : resolve()) + }).finally(checkDiags) + } + + function waitForResult() { + return new Promise((resolve, reject) => { + stateEmitter.once('result', (data?: any, err?: Error) => err ? reject(err) : resolve(data)) + }).finally(checkDiags) + } + + let buffer = '' + function onData(d: Buffer) { + const lines = (buffer + d).split('\n') + buffer = lines.pop() ?? '' + + for (const l of lines) { + try { + const msg = JSON.parse(l) as TfJsonOutput | { type: 'ready' } // TODO: remove `ready` and just use `result` + + if (msg.type === 'ready') { + isReady = true + stateEmitter.emit('ready') + stateEmitter.emit('result', undefined, undefined) + } else if (msg.type === 'result') { + stateEmitter.emit('result', msg.data) + } else { + logger(msg) + if (msg.type === 'plan') { + stateEmitter.emit('plan', msg.data) + } else if (msg.type === 'diagnostic') { + + } + } + } catch(e) { + getLogger().debug('Bad parse', e) + // Failure is due to non-JSON data + } + } + } + + // TODO: try to kill process on cancel + + c.stdout.on('data', onData) + c.stderr.on('data', d => getLogger().raw(d)) + + function write(chunk: string | Buffer) { + return new Promise((resolve, reject) => { + c.stdin.write(chunk, err => err ? reject(err) : resolve()) + }) + } + + await waitForReady() + + async function setTemplate(template: TfJson) { + await getFs().writeFile(templateFilePath, JSON.stringify(template)) + isReady = false + await write(`${['reload-config'].join(' ')}\n`) + + return waitForReady() + } + + return { + setTemplate, + getRefs: async (targets: string[]) => { + isReady = false + const result = waitForResult>() + await write(`${['get-refs', ...targets].join(' ')}\n`) + await waitForReady() + + return result + }, + dispose: () => write('exit\n'), + } +} + +function getDeployOptionsArgs(opt: DeployOptions | undefined) { + const args: string[] = [] + if (!opt) { + return args + } + + if (opt.parallelism) { + args.push(`-parallelism=${opt.parallelism}`) + } + + if (opt.disableRefresh) { + args.push('-refresh=false') + } + + if (opt.useTests) { + args.push('-use-tests') + } + + if (opt.autoApprove) { + args.push('-auto-approve') + } + + function addMultiValuedSwitch(switchName: string, val: string | string[]) { + if (!Array.isArray(val)) { + args.push(`-${switchName}=${val}`) + + return + } + + for (const r of val) { + args.push(`-${switchName}=${r}`) + } + } + + if (opt.targetResources) { + addMultiValuedSwitch('target', opt.targetResources) + } + + if (opt.targetFiles) { + addMultiValuedSwitch('module', opt.targetFiles) + } + + if (opt.replaceResource) { + addMultiValuedSwitch('replace', opt.replaceResource) + } + + return args +} + +interface TerraformVersion { + readonly version: string + readonly platform: string //"darwin_arm64", + // "provider_selections": { + // "registry.terraform.io/hashicorp/test1": "7.8.9-beta.2", + // "registry.terraform.io/hashicorp/test2": "1.2.3" + //} +} + +async function getTerraformVersion(tfPath: string) { + const stdout = await runCommand(tfPath, ['-v', '-json']) + const parsed = JSON.parse(stdout) + if ('terraform_version' in parsed) { + throw new Error(`Binary is not compatible with Synapse`) + } + + if (typeof parsed.version !== 'string' || typeof parsed.platform !== 'string') { + throw new Error('Bad output format') + } + + return parsed +} + +export interface TfStateResource { + // mode: 'managed' + readonly type: string + readonly name: string + readonly provider: string + readonly values: Record +} + +// export async function handleErrorState(platform: Platform, opt?: DeployOptions) { +// const templateService = platform.getTemplateService() +// const templateFile = await templateService.getTemplateFilePath() +// const errorFilename = path.resolve(path.dirname(templateFile), 'errored.tfstate') + +// try { +// await Promise.all([ +// fs.access(errorFilename, fs.constants.R_OK), +// fs.access(path.dirname(errorFilename), fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK), +// ]) + +// getLogger().log('Restoring error state:', errorFilename) +// await runTerraformCommand(platform, 'state', ['push', path.basename(errorFilename)], opt) + +// getLogger().log('Error state restored! Deleting the file.') +// await fs.unlink(errorFilename) +// } catch (e) { +// if ((e as any).code !== 'ENOENT') { +// throw e +// } +// } +// } + + +// TODO: after successful destroy, delete these files +// .terraform/terraform.tfstate +// .terraform.lock.hcl + +interface TfJsonOutputBase { + '@level': 'info' | 'warn' | 'error' + '@message': string + '@module': string // always `terraform.ui` + '@timestamp': string // RFC3339 +} + +type TfAction = 'create' | 'read' | 'update' | 'replace' | 'delete' // | 'noop' | + +interface TfLog extends TfJsonOutputBase { + type: 'log' +} + +interface TfResult extends TfJsonOutputBase { + type: 'result' + data: any +} + +interface TfChangeSummary extends TfJsonOutputBase { + type: 'change_summary' + changes: { + add: number + change: number + import: number + remove: number + operation: 'plan' | 'apply' | 'destroy' // Not sure if these are correct + } +} + +interface TfResourceDrift extends TfJsonOutputBase { + type: 'resource_drift' + change: { + resource: TfResource, + action: 'update' // ??? + } +} + +// This message does not include details about the exact changes which caused the change to be planned. +// That information is available in the JSON plan output. +interface TfPlannedChange extends TfJsonOutputBase { + type: 'planned_change' + change: { + resource: TfResource + previous_resource?: TfResource + action: TfAction | 'move' + reason?: 'tainted' | 'requested' | 'cannot_update' | 'delete_because_no_resource_config' | 'delete_because_wrong_repetition' | 'delete_because_count_index' | 'delete_because_each_key' | 'delete_because_no_module' + } +} + +interface TfRefreshStart extends TfJsonOutputBase { + type: 'refresh_start' + hook: { + resource: TfResource + id_key: string + id_value: string + } +} + +interface TfRefreshComplete extends TfJsonOutputBase { + type: 'refresh_complete' + hook: { + resource: TfResource + id_key: string + id_value: string + } +} + +interface TfApplyStart extends TfJsonOutputBase { + type: 'apply_start' + hook: { + action: TfAction + resource: TfResource + id_key?: string + id_value?: string // maybe string + } +} + +interface TfApplyProgress extends TfJsonOutputBase { + type: 'apply_progress' + hook: { + action: TfAction + resource: TfResource + id_key?: string + id_value?: unknown + elapsed_seconds: number + } +} + +interface TfApplyComplete extends TfJsonOutputBase { + type: 'apply_complete' + hook: { + action: TfAction + resource: TfResource + id_key?: string + id_value?: unknown + elapsed_seconds: number + state?: TfState['resources'][number] + } +} + +interface TfResource { + addr: string + module: string + resource: string + resource_type: string + resource_name: string + resource_key: string | null + implied_provider: string +} + +// Error is rendered as a diagnostic... +interface TfApplyErrored extends TfJsonOutputBase { + type: 'apply_errored' + hook: { + action: TfAction + resource: TfResource + id_key?: string + id_value?: unknown + elapsed_seconds: number + reason?: string + } +} + +interface TfDiagnostic { + severity: 'warning' | 'error' + summary: string + detail?: string + range?: { + filename: string + start: TfPosition // inclusive + end: TfPosition // exclusive + }, + snippet?: { + context?: string + code: string + start_line: number + highlight_start_offset: number + highlight_end_offset: number + values: { + traversal: string + statement: string + }[] + } +} + +interface TfDiagnosticOutput extends TfJsonOutputBase { + type: 'diagnostic' + valid: boolean + error_count: number + warning_count: number + diagnostic?: TfDiagnostic + diagnostics?: TfDiagnostic[] +} + +interface InstallProviderEvent { + address: string + name: string + version: string + size?: number + downloaded?: number + phase?: 'downloading' | 'verifying' | 'extracting' | 'complete' + error?: string +} + +interface TfInstallProvider extends TfJsonOutputBase { + type: 'install_provider' + hook: InstallProviderEvent +} + +interface TfPosition { + byte: number + line: number + column: number +} + +interface TfErrorOutput extends TfJsonOutputBase { + type: 'error' + data: string +} + +interface TfPlanOutput extends TfJsonOutputBase { + type: 'plan' + data: Omit +} + +type TfJsonOutput = + | TfLog + | TfResult + | TfPlanOutput + | TfErrorOutput + | TfChangeSummary + | TfResourceDrift + | TfPlannedChange + | TfRefreshStart + | TfRefreshComplete + | TfApplyStart + | TfApplyProgress + | TfApplyComplete + | TfApplyErrored + | TfInstallProvider + | TfDiagnosticOutput + +// https://developer.hashicorp.com/terraform/internals/json-format#plan-representation +type ActionReason = + | 'read_because_config_unknown' + | 'delete_because_no_resource_config' + | 'replace_by_triggers' + | 'replace_because_cannot_update' + +interface TfPlan { + prior_state?: { + format_version: string // ??? not sure + terraform_version: string + values: { + root_module: { + resources: { + address: string + type: string + name: string + values: Record + depends_on?: string[] + }[] + } + } + } + resource_changes?: { + readonly type: string + readonly name: string + readonly action_reason: ActionReason + readonly change: TfResourceChange + }[] + relevant_attributes?: { + readonly resource: string + readonly attribute: string[] + }[] +} + +interface TfResourceChange { + readonly actions: ('no-op' | 'create' | 'read' | 'update' | 'delete')[] + readonly before: any + readonly after: any + readonly after_unknown: TfResourceChange['after'] // But with booleans + readonly replace_paths?: string[][] +} + +function filterNoise(plan: TfPlan) { + const filtered = plan.resource_changes?.filter(r => { + if (r.change.actions.length === 1 && ['no-op', 'read'].includes(r.change.actions[0])) { + return false + } + + return true + }) + + return { ...plan, resource_changes: filtered } +} + +const unknown = Symbol.for('tfUnknown') + +function mergeUnknowns(val1: any, val2: any): any { + if (val2 === true) { + return unknown + } + + if (Array.isArray(val1)) { + if (!Array.isArray(val2)) { + throw new Error(`Expected arrays: ${val1} -> ${val2}`) + } + + return val1.map((x, i) => mergeUnknowns(x, val2[i])) + } + + if (typeof val1 === 'object' && typeof val2 === 'object' && val1 !== null && val2 !== null) { + const res: Record = {} + const keys = new Set([...Object.keys(val1), ...Object.keys(val2)]) + for (const k of keys) { + const v1 = val1[k] + const v2 = val2[k] + if (v2 === undefined) { + res[k] = v1 + } else { + res[k] = mergeUnknowns(v1, v2) + } + } + + return res + } + + + return val1 +} + +// ASSUMES KEYS ARE ALREADY SORTED +function diff(val1: any, val2: any): any { + if (val1 === val2) { + return + } + + if (typeof val1 !== typeof val2 || typeof val1 !== 'object' || val1 === null || val2 === null) { + return { from: val1, to: val2 } + } + + if (Array.isArray(val1)) { + if (!Array.isArray(val2)) { + throw new Error(`Expected arrays: ${val1} -> ${val2}`) + } + + const length = Math.max(val1.length, val2.length) + const res: any[] = [] + for (let i = 0; i < length; i++) { + const v1 = i >= val1.length ? null : val1[i] + const v2 = i >= val2.length ? null : val2[i] + const d = diff(v1, v2) + res.push(d) + } + + if (res.filter(x => x !== undefined).length !== 0) { + return res + } + + return undefined + } + + const res: Record = {} + const keys = new Set([...Object.keys(val1), ...Object.keys(val2)]) + for (const k of keys) { + if (val2[k] === undefined) continue + + const d = diff(val1[k], val2[k]) + if (d !== undefined) { + res[k] = d + } + } + + if (Object.keys(res).length > 0) { + return res + } +} + +interface ResourcePlan { + readonly change: TfResourceChange + readonly reason?: ActionReason + readonly attributes?: string[][] + readonly state?: any +} + +export function getChangeType(change: TfResourceChange): 'create' | 'update' | 'replace' | 'delete' | 'read' | 'no-op' { + if (change.actions.length === 1) { + return change.actions[0] + } + + if (change.actions[0] === 'create' && change.actions[1] === 'delete') { + return 'replace' + } + + if (change.actions[0] === 'delete' && change.actions[1] === 'create') { + return 'replace' + } + + throw new Error(`Unknown change action set: ${change.actions.join(', ')}`) +} + +export function isTriggeredReplaced(plan: ResourcePlan) { + if (getChangeType(plan.change) !== 'replace') { + return false + } + + return plan.reason === 'replace_by_triggers' +} + +export function getDiff(change: TfResourceChange): any { + return diff(change.before, mergeUnknowns(change.after ?? {}, change.after_unknown)) +} + +export type ParsedPlan = Record +export function parsePlan(plan: Omit, state?: Record): ParsedPlan { + const resources = plan.resource_changes ?? [] + const relevantAttributes: Record = {} + if (plan.relevant_attributes) { + for (const o of plan.relevant_attributes) { + const arr = relevantAttributes[o.resource] ??= [] + arr.push(o.attribute) + } + } + + return Object.fromEntries( + resources.map( + r => [`${r.type}.${r.name}`, { + change: r.change, + reason: r.action_reason, + attributes: relevantAttributes[`${r.type}.${r.name}`], + state: state?.[`${r.type}.${r.name}`], + }] as const + ) + ) +} + +export function renderPlan(plan: TfPlan) { + const filtered = filterNoise(plan) + const values = filtered.prior_state?.values + const states = values ? Object.fromEntries(values.root_module.resources.map(r => [r.address, r])) : undefined + + return parsePlan(filtered, states) +} + +export async function getTerraformPath() { + // This is configured on installation. + const configuredPath = await readKey('terraform.path') + if (configuredPath) { + return configuredPath + } + + throw new Error(`Missing binary. Corrupted installation?`) + + // const respCache = createTtlCache(createMemento(getFs(), path.resolve(getSynapseDir(), 'memento'))) + // const cachedPath = await respCache.get('tf-install-path') + // if (cachedPath) { + // return cachedPath + // } + + // const lockPath = path.resolve(getSynapseDir(), 'tf-installer') + // await using _ = await acquireFsLock(lockPath) + + // const installed = await getOrInstallTerraform(getSynapseDir()) + // await respCache.set('tf-install-path', installed.path, 300) + + // return installed.path as string +} + +// We mutate the state object directly +export function createStatePersister(currentState: TfState | undefined, programHash: string, procFs = getDeploymentFs()) { + const getLineage = memoize(() => currentState?.lineage ?? randomUUID()) + const getNextSerial = memoize(() => (currentState?.serial ?? 0) + 1) + + function createStateFile(resources: TfState['resources']): TfState { + const version = resources.length === 0 + ? (currentState?.version ?? 4) + : resources[0].state ? 5 : 4 + + return { + version, + serial: getNextSerial(), + lineage: getLineage(), + resources, + } + } + + // These hashes are used for incremental deploys + const resourceHashes: Record = {} + const stateMap: Record = {} + if (currentState) { + for (const r of currentState.resources) { + stateMap[`${r.type}.${r.name}`] = r + } + } + + const getPreviousHashes = memoize(() => getResourceProgramHashes(procFs)) + + async function saveHashes() { + const previous = await getPreviousHashes() + const merged = sortRecord({ ...previous, ...resourceHashes }) + for (const [k, v] of Object.entries(merged)) { + if (v === null) { + delete merged[k] + } + } + await setResourceProgramHashes(procFs, merged as Record) + } + + async function _saveState() { + await Promise.all([ + saveHashes(), + putState(createStateFile(Object.values(stateMap)), procFs), + ]) + } + + let writeTimer: number | undefined + let pendingSave: Promise | undefined + function saveState() { + return pendingSave ??= _saveState().finally(() => pendingSave = undefined) + } + + function triggerSave() { + clearTimeout(writeTimer) + writeTimer = +setTimeout(async () => { + await pendingSave + await saveState() + }, 10) + } + + function updateResource(id: string, instanceState?: TfState['resources'][number]) { + if (!instanceState) { + delete stateMap[id] + } else { + stateMap[id] = instanceState + } + + triggerSave() + } + + const l = getLogger().onDeploy(ev => { + if (ev.action === 'noop') { + resourceHashes[ev.resource] = programHash + } + if (ev.status !== 'complete' || ev.resource.startsWith('data.')) return + + if (ev.action !== 'read' && ev.action !== 'noop') { + updateResource(ev.resource, ev.state) + } + + if (ev.action !== 'read') { + if (ev.action === 'delete') { + resourceHashes[ev.resource] = null + } else { + resourceHashes[ev.resource] = programHash + } + } + }) + + const l2 = getLogger().onPlan(ev => { + let needsSave = false + for (const [k, v] of Object.entries(ev.plan)) { + if (k.startsWith('data.')) continue + + const change = getChangeType(v.change) + if (change === 'no-op') { + resourceHashes[k] = programHash + needsSave = true + } + } + + if (needsSave) { + triggerSave() + } + }) + + async function dispose() { + l.dispose() + l2.dispose() + clearTimeout(writeTimer) + + if (writeTimer && !pendingSave) { + await saveState() + } else { + await pendingSave + } + } + + return { getLineage, getNextSerial, dispose } +} \ No newline at end of file diff --git a/src/deploy/httpServer.ts b/src/deploy/httpServer.ts new file mode 100644 index 0000000..f7340cc --- /dev/null +++ b/src/deploy/httpServer.ts @@ -0,0 +1,376 @@ +import type * as http from 'node:http' +// import * as http2 from 'http2' +import * as url from 'node:url' +import * as stream from 'node:stream' +import { getLogger } from '..' + +// https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods +type HttpMethod = 'GET' | 'POST' | 'PUT' | 'HEAD' | 'DELETE' | 'PATCH' | string + +type TrimRoute = T extends `${infer U}+` ? U : T +type ExtractPattern = T extends `${infer P}{${infer U}}${infer S}` ? TrimRoute | ExtractPattern

: never +type CapturedPattern = { [P in ExtractPattern]: string } +type RouteHandler = (request: HttpRequest) => any + +// /cars?color=blue&brand=ferrari + +// type ExtractQuery = T extends `${infer K}=${infer V}&${infer R}` +// ? [[K, V], ...ExtractQuery] +// : T extends `${infer K}=${infer V}` ? [[K, V]] : [] + +// type CapturedQuery = { [P in ExtractPattern]: string } + + +interface BaseHttpRequest { + readonly request: http.IncomingMessage + readonly path: T + readonly method: HttpMethod + readonly params: CapturedPattern + readonly query: string + readonly queryParams: Record +} + +interface HttpRequest extends BaseHttpRequest { + readonly response: http.ServerResponse +} + +interface HttpUpgrade extends BaseHttpRequest { + readonly socket: stream.Duplex + readonly head: Buffer +} + +interface StructuredHttpRequest> extends HttpRequest { + readonly body: U +} + + +interface Route { + readonly path: T + handleMessage(message: BaseHttpRequest): U +} + +export class HttpRoute = RouteHandler> { + public constructor(public readonly path: T, private readonly handler: U) { + + } + + public handleMessage(request: HttpRequest) { + return this.handler(request) + } +} + +interface HttpErrorProps { + readonly code?: string + readonly statusCode?: number +} + +export class HttpError extends Error { + public readonly code: string + public readonly statusCode: number + + public constructor(message: string, private readonly props?: HttpErrorProps) { + super(message) + this.code = this.props?.code ?? 'UnknownError' + this.statusCode = this.props?.statusCode ?? 500 + } + + public serialize() { + return JSON.stringify({ + code: this.code, + message: this.message, + stack: this.stack, + }) + } + + public static cast(e: unknown) { + if (e instanceof this) { + return e + } + + if (e instanceof Error) { + return Object.assign(new this(e.message), { stack: e.stack }) + } + + return new this(`Unknown error: ${e}`) + } +} + +function getHttp() { + return require('node:http') as typeof import('node:http') +} + +export class HttpServer { + #requestCounter = 0 + readonly #server = getHttp().createServer() + readonly #routes: Route[] = [] + readonly #patterns = new Map>() + readonly #pending = new Map>() + readonly #errors = new Map() + + public constructor(private readonly options?: { readonly port: number }) {} + + public static fromRoutes(...routes: U): HttpServer { // & ConvertRoutes { + const instance = new this() + instance.#routes.push(...routes) + routes.forEach(r => instance.#patterns.set(r, buildRouteRegexp(r.path))) + + return instance + } + + public takeError(requestId: string): unknown | undefined { + const err = this.#errors.get(requestId) + if (err === undefined) { + return + } + + this.#errors.delete(requestId) + return err + } + + public async start(port?: number, hostname = 'localhost') { + return new Promise((resolve, reject) => { + this.#server.on('error', reject) + this.#server.listen(port ?? this.options?.port, hostname, () => { + const addr = this.#server.address() + if (typeof addr === 'string' || !addr) { + reject(new Error(`Unexpected server address: ${JSON.stringify(addr)}`)) + } else { + resolve(addr.port) + } + }) + + this.#server.on('request', (req, res) => { + function emitError(e: unknown, requestId: string) { + const err = HttpError.cast(e) + const blob = err.serialize() + + res.writeHead(err.statusCode, { + 'content-type': 'application/json', + 'content-length': blob.length, + 'x-synapse-request-id': requestId, + }) + res.end(blob) + } + + const handleRequest = () => { + if (!req.url) { + throw new HttpError(`No url: ${req}`) + } + + const { pathname, query } = url.parse(req.url) + if (!pathname) { + throw new HttpError(`No pathname: ${req.url}`) + } + + const result = this.matchRoute(pathname) + if (!result) { + throw new HttpError(`No route found: ${pathname}`) + } + + if (!(result.route instanceof HttpRoute)) { + throw new HttpError(`Not an http route: ${result.route.path}`) + } + + return result.route.handleMessage({ + request: req, + path: pathname, + params: result.match.groups, + query: query ?? '', + queryParams: {}, + method: (req.method ?? 'unknown') as any, + response: res, + }) + } + + const requestId = this.#requestCounter += 1 + const p = handleRequest().catch((e: any) => { + const rId = `${requestId}` + this.#errors.set(rId, e) + emitError(e, rId) + }) + + this.#pending.set(requestId, p.finally(() => this.#pending.delete(requestId))) + }) + }) + } + + public close(): Promise { + return new Promise(async (resolve, reject) => { + await Promise.all(this.#pending.values()) + + this.#server.close(err => err ? reject(err) : resolve()) + }) + } + + private matchRoute(path: string) { + const results = this.#routes.map(route => { + const pattern = this.#patterns.get(route)! + const match = pattern.exec(path) + + if (match) { + return { route, match } + } + }) + + return results.find(r => !!r) + } +} + +interface TypedRegExpExecArray> extends RegExpExecArray { + readonly groups: T +} + +interface RouteRegexp extends RegExp { + exec(string: string): TypedRegExpExecArray> | null +} + +function buildRouteRegexp(path: T): RouteRegexp { + const pattern = /{([A-Za-z0-9]+\+?)}/g + const searchPatterns: string[] = [] + let match: RegExpExecArray | null + let lastIndex = 0 + + // TODO: handle duplicates + while (match = pattern.exec(path)) { + const isGreedy = match[1].endsWith('+') + const name = isGreedy ? match[1].slice(0, -1) : match[1] + + searchPatterns.push(path.slice(lastIndex, match.index - 1)) + if (!isGreedy) { + searchPatterns.push(`\\/(?<${name}>[^\\/\\s:]+\\/?)`) + } else { + searchPatterns.push(`\\/(?<${name}>[^\\s:]+)`) + } + lastIndex = pattern.lastIndex + } + + if (lastIndex < path.length) { + searchPatterns.push(path.slice(lastIndex)) + } + + searchPatterns.push('$') + + return new RegExp('^' + searchPatterns.join('')) as RouteRegexp +} + +export function receiveData(message: http.IncomingMessage): Promise { + const data: any[] = [] + + return new Promise((resolve, reject) => { + message.on('error', reject) + message.on('data', chunk => data.push(chunk)) + message.on('end', () => resolve(data.join(''))) + }) +} + +export function sendResponse(response: http.ServerResponse, data?: any): Promise { + if (data === undefined) { + return new Promise((resolve, reject) => { + response.on('error', reject) + response.writeHead(204) + response.end(resolve) + }) + } + + const contentType = typeof data === 'object' ? 'application/json' : 'application/octet-stream' + const blob = typeof data === 'object' ? Buffer.from(JSON.stringify(data), 'utf-8') : Buffer.from(data) + + return new Promise((resolve, reject) => { + response.on('error', reject) + response.writeHead(200, { + 'Content-Type': contentType, + 'Content-Length': blob.length, + }) + response.end(blob, resolve) + }) +} + +type WebSocketRouteHandler = (params: CapturedPattern, socket: WebSocket) => any + + + +// type Http2RouteHandler = (stream: Http2Stream) => any + +// interface Http2Stream { +// readonly params: CapturedPattern +// readonly stream: http2.ServerHttp2Stream +// } + +// export class Http2Route = Http2RouteHandler> { +// public constructor(public readonly path: T, private readonly handler: U) { + +// } + +// public handleMessage(stream: Http2Stream) { +// this.handler(stream) +// } +// } + +// interface Http2Route2 { +// readonly path: T +// handleMessage(stream: Http2Stream): any +// } + + +// export class Http2Server { +// readonly #routes: Http2Route2[] = [] +// readonly #patterns = new Map>() + +// public constructor(private readonly options?: { readonly port: number }) { + +// } + +// public static fromRoutes(...routes: U): Http2Server { +// const instance = new this() +// instance.#routes.push(...routes) +// routes.forEach(r => instance.#patterns.set(r, buildRouteRegexp(r.path))) + +// return instance +// } + +// public listen(port = this.options?.port ?? 80) { +// this.server.listen(port) +// this.server.on('stream', (stream, headers) => { +// function destroy(err?: Error): never { +// stream.destroy(err) +// throw err +// } + +// const method = headers[http2.constants.HTTP2_HEADER_METHOD] +// const path = headers[http2.constants.HTTP2_HEADER_PATH] +// if (typeof method !== 'string') { +// destroy(new Error(`Method was not a string: ${method}`)) +// } else if (typeof path !== 'string') { +// destroy(new Error(`Path was not a string: ${path}`)) +// } + +// const result = this.matchRoute(path) +// if (!result) { +// destroy(new Error(`No route found: ${path}`)) +// } + +// result.route.handleMessage({ +// stream, +// params: result.match.groups, +// }) +// }) +// } + +// #server?: http2.Http2Server +// private get server() { +// return this.#server ??= http2.createServer() +// } + +// private matchRoute(path: string) { +// const results = this.#routes.map(route => { +// const pattern = this.#patterns.get(route)! +// const match = pattern.exec(path) + +// if (match) { +// return { route, match } +// } +// }) + +// return results.find(r => !!r) +// } +// } diff --git a/src/deploy/registry.ts b/src/deploy/registry.ts new file mode 100644 index 0000000..bf80232 --- /dev/null +++ b/src/deploy/registry.ts @@ -0,0 +1,106 @@ +import { randomUUID } from 'node:crypto' +import { memoize } from '../utils' +import { DeploymentContext, ProviderRequest, assertNotData, loadPointer } from './server' +import type { TfState } from './state' +import { getSynapseResourceInput, getSynapseResourceOutput, getSynapseResourceType } from './deployment' + +export interface ServiceProvider { + readonly kind: string + load(id: string, config: T): Promise | void + unload(id: string): Promise | void +} + + +export const getServiceRegistry = memoize(createServiceRegistry) +export const apiRegistrationResourceType = 'ApiRegistration' + +function createServiceRegistry() { + const services = new Map() + const registered = new Map() + + function getServiceProvider(kind: string) { + const provider = services.get(kind) + if (!provider) { + throw new Error(`Missing service for API handler: ${kind}`) + } + return provider + } + + function registerServiceProvider(provider: ServiceProvider) { + if (services.has(provider.kind)) { + throw new Error(`Service already registered: ${provider.kind}`) + } + services.set(provider.kind, provider) + } + + async function setRegistration(id: string, kind: string, config: any) { + const provider = getServiceProvider(kind) + console.log(id, config) + await provider.load(id, config) + registered.set(id, config) + } + + async function deleteRegistration(id: string, kind: string) { + if (!registered.has(id)) { + return + } + + const provider = getServiceProvider(kind) + await provider.unload(id) + registered.delete(id) + } + + async function createRegistration(ctx: DeploymentContext, request: ProviderRequest & { operation: 'create' } ) { + const id = randomUUID() + const config = await loadPointer(ctx.createModuleLoader(), request.plannedState.config) + await setRegistration(id, request.plannedState.kind, config) + + return { id } + } + + async function handleRequest(ctx: DeploymentContext, request: ProviderRequest) { + assertNotData(request.operation) + + switch (request.operation) { + case 'create': + return createRegistration(ctx, request) + + case 'update': + const id = request.priorState.id + const config = await loadPointer(ctx.createModuleLoader(), request.plannedState.config) + await deleteRegistration(id, request.priorInput.kind) + await setRegistration(id, request.plannedState.kind, config) + + return { id } + + case 'delete': + return deleteRegistration(request.priorState.id, request.priorInput.kind) + + case 'read': + return request.priorState + } + } + + async function loadFromState(ctx: DeploymentContext, state: TfState) { + const resources = state.resources + .filter(x => getSynapseResourceType(x) === apiRegistrationResourceType) + .map(x => [getSynapseResourceOutput(x).id, getSynapseResourceInput(x)]) + + const promises: Promise[] = [] + for (const [id, { kind, config }] of resources) { + async function doResolve() { + const resolved = await loadPointer(ctx.createModuleLoader(), config) + await setRegistration(id, kind, resolved) + } + promises.push(doResolve()) + } + await Promise.all(promises) + } + + return { + handleRequest, + registerServiceProvider, + + loadFromState, + } +} diff --git a/src/deploy/server.ts b/src/deploy/server.ts new file mode 100644 index 0000000..19479d8 --- /dev/null +++ b/src/deploy/server.ts @@ -0,0 +1,1061 @@ +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import * as child_process from 'node:child_process' +import { HttpServer, HttpRoute, HttpError, sendResponse, receiveData } from './httpServer' +import { objectId, resolveValue } from '../runtime/modules/serdes' +import { BinaryToTextEncoding, createHash, randomUUID } from 'node:crypto' +import { bundleClosure, createDataExport, getImportMap, isDeduped, normalizeSymbolIds } from '../closures' +import { getLogger } from '../logging' +import { ModuleResolver, createImportMap } from '../runtime/resolver' +import { AsyncLocalStorage } from 'async_hooks' +import { BuildFsFragment, ProcessStore } from '../artifacts' +import { PackageService } from '../pm/packages' +import { TerraformPackageManifest } from '../runtime/modules/terraform' +import { Fs, SyncFs, readDirRecursive } from '../system' +import { Pointers, createPointer, isDataPointer, pointerPrefix } from '../build-fs/pointers' +import { ImportMap, SourceInfo } from '../runtime/importMaps' +import { getWorkingDir } from '../workspaces' +import { apiRegistrationResourceType, getServiceRegistry } from './registry' + + +export interface DeploymentContext { + readonly fs: Fs & SyncFs + readonly processStore: ProcessStore + readonly dataDir: string + readonly packageManifest: TerraformPackageManifest + readonly packageService: PackageService + createModuleResolver(): ModuleResolver + createModuleLoader(): ModuleLoader + createZip?(files: Record, dest: string): Promise +} + +export interface ModuleLoader { + loadModule: (id: string, origin?: string) => Promise + registerMapping(mapping: ImportMap, location: string): void + runWithContext: (namedContexts: Record, fn: () => Promise | T) => Promise +} + + +interface TerraformResourceConfig { + readonly type: string + readonly plan: any + readonly context?: any + readonly handler: string +} + +interface TerraformResourceRequest extends TerraformResourceConfig { + readonly resource: string + readonly state: any + readonly operation: 'create' | 'update' | 'read' | 'delete' + readonly workingDirectory: string + + // Only relevant for `update` + readonly priorConfig?: TerraformResourceConfig +} + +interface ResourceDefinition< + D extends object = object, + T extends object = object, + I extends object = T, + U extends any[] = [], +> { + read?(state: I): T | Promise + create?(...args: U): T | Promise + update?(state: T, ...args: U): T | Promise + delete?(state: T, ...args: U): void | Promise + data?(state: I): D | Promise +} + +export async function loadPointer(loader: ModuleLoader, val: any) { + val = await resolvePointer(val) + + return val === undefined ? val : deserializeObject(loader, val) +} + +async function resolvePointer(val: any) { + if (val === undefined) { + return + } + + if (isDataPointer(val)) { + return getObject(val) + } + + if (typeof val === 'string' && val.startsWith(pointerPrefix)) { + return getObject(val) + } + + return val +} + +function deserializeObject(loader: ModuleLoader, obj: any) { + if (isDeduped(obj)) { + return resolveValue(obj.captured, loader, obj.table, undefined, false) + } + + return resolveValue(obj, loader, undefined, undefined, false) +} + +function createProviderRoutes(ctx: DeploymentContext) { + const ops = ['read', 'create', 'update', 'delete', 'data'] + async function resolveConfig(loader: ModuleLoader, config: TerraformResourceConfig, resourceName: string) { + if (!config.handler.startsWith(pointerPrefix)) { + throw new Error(`Unexpected legacy handler found while resolving config: ${config.handler}`) + } + + const handler = config.handler + + // We have to load the plan before we load the handler so we can parse + // out any package dependencies that might have been serialized + const plan = await loadPointer(loader, config.plan) ?? [] as any[] + const tabularized = createDataExport(plan) + const importMap = await getImportMap(ctx, tabularized.table) + const pointerMappings = await ctx.packageService.getPublishedMappings('[handler]', handler) + + if (importMap || pointerMappings) { + const m = importMap ?? {} + if (pointerMappings) { + m[pointerMappings[0]] = pointerMappings[1] + } + getLogger().debug('Registering import map parsed from resource plan:', handler, Object.keys(m)) + loader.registerMapping(m, getWorkingDir()) + } + + const definition: ResourceDefinition = await loader.loadModule(handler) + if (!definition) { + throw new HttpError(`No resource definition found for type: ${config.type} (${config.handler})`, { + statusCode: 404, + code: 'NoResourceDefinition', + }) + } + + if (!ops.some(o => o in definition)) { + throw new HttpError(`Resource definition contains no operations: ${config.type} (${config.handler})`, { + statusCode: 400, + code: 'BadResourceDefinition', + }) + } + + const context = await loadPointer(loader, config.context) ?? {} + const afs = await getArtifactFs() + context['afs'] ??= [] + context['afs'].push(afs) + + context['resource'] ??= [] + context['resource'].push(resourceName) + + return { + plan, + context, + definition, + // dataTable: (await resolvePointer(config.plan))?.table ?? {}, + } + } + + async function resolvePayload(loader: ModuleLoader, req: TerraformResourceRequest) { + const config = await resolveConfig(loader, req, req.resource) + const state = await loadPointer(loader, req.state) ?? {} + + return { + state, + ...config, + } + } + + async function handleProviderRequest(request: TerraformResourceRequest) { + const loader = ctx.createModuleLoader() + const resolved = await resolvePayload(loader, request) + const { state, definition, plan, context } = resolved + + async function deleteResource() { + const priorResolved = request.priorConfig + ? await resolveConfig(loader, request.priorConfig, request.resource) + : undefined + + if (priorResolved) { + const deleteOp = priorResolved.definition['delete'] + await loader.runWithContext(priorResolved.context, async () => { + await (deleteOp as any)?.(state, ...priorResolved.plan) + }) + + return + } + + const deleteOp = definition['delete'] + await (deleteOp as any)?.(state, ...plan) + } + + const op = definition[request.operation] + async function doOp() { + if (!op && request.operation === 'update') { + await deleteResource() + const createOp = definition['create'] + + const newState = (await (createOp as any)?.(...plan)) ?? state + + return newState + } + + if (request.operation === 'delete') { + await deleteResource() + + return {} + } + + const parameters = request.operation === 'create' ? plan + : request.operation === 'update' ? [state, ...plan] : [state] + + const newState = (await (op as any)?.(...(parameters))) ?? state + + return newState + } + + return await loader.runWithContext(context, doOp) + } + + async function handleDataProviderRequest(request: TerraformResourceRequest) { + const loader = ctx.createModuleLoader() + const { definition, plan, context } = await resolvePayload(loader, request) + if (request.operation !== 'read') { + throw new Error(`Unexpected operation: ${request.operation}. Only 'read' is allowed.`) + } + + async function doOp() { + const op = definition['data'] ?? definition['read'] + if (!op) { + getLogger().log('No "data" or "read" method found.') + + return plan[0] ?? {} + } + + const state = (await (op as any)(...plan)) + + return state + } + + return await loader.runWithContext(context, doOp) + } + + function serializeValue(val: any, dataTable: Record): any { + if ((typeof val !== 'object' && typeof val !== 'function') || !val) { + return val + } + + if (isDataPointer(val)) { + return val + } + + const id = val[objectId] + if (id !== undefined) { + return { ['@@__moveable__']: dataTable[id] } + } + + if (Array.isArray(val)) { + return val.map(v => serializeValue(v, dataTable)) + } + + if (typeof val === 'function') { + throw new Error(`Failed to serialize value: ${val}`) + } + + // TODO: fail on prop descriptors + + const o: Record = {} + for (const [k, v] of Object.entries(val)) { + o[k] = serializeValue(v, dataTable) + } + + return o + } + + return { + handleProviderRequest, + handleDataProviderRequest, + } +} + +async function ensureDir(fileName: string) { + try { + await fs.mkdir(path.dirname(fileName), { recursive: true }) + } catch(e) { + if ((e as any).code !== 'EEXIST') { + throw e + } + } +} + +function createStateRoute(stateDirectory: string) { + const getStatePath = (org: string, workspace: string, branch: string, module: string) => + path.resolve(stateDirectory, org, workspace, branch, module) + + async function saveState(location: string, data: string) { + await ensureDir(location) + await fs.writeFile(location, data, 'utf-8') + } + + async function loadState(location: string) { + try { + const data = await fs.readFile(location, 'utf-8') + + return data + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + + throw new HttpError('No state found', { code: 'MissingState', statusCode: 404 }) + } + } + + async function deleteState(location: string) { + try { + await fs.unlink(location) + } catch(e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } + } + + const stateRoute = new HttpRoute('/{org}/{workspace}/{branch}/state/{module+}', async request => { + const { org, workspace, branch, module } = request.params + const location = getStatePath(org, workspace, branch, module) + + switch (request.method) { + case 'GET': + return sendResponse(request.response, await loadState(location)) + case 'POST': + return sendResponse(request.response, await saveState(location, await receiveData(request.request))) + case 'DELETE': + return sendResponse(request.response, await deleteState(location)) + + // LOCK + // UNLOCK + } + + throw new HttpError(`Invalid method: ${request.method}`, { code: 'InvalidMethod', statusCode: 400 }) + }) + + return [stateRoute] +} + +type Operation = 'create' | 'update' | 'read' | 'delete' | 'data' + +export function assertNotData(op: Operation): asserts op is Exclude { + if (op === 'data') { + throw new Error('Data operations are not allowed for resources') + } +} + +async function createArtifactFs2(ctx: DeploymentContext, resource: string, deps: string[], op: Operation) { + const sfs = op !== 'create' && op !== 'update' && op !== 'data' + ? await ctx.processStore.getResourceStore(resource) + : await ctx.processStore.createResourceStore(resource, op === 'update' ? [...deps, resource] : deps) + + return sfs +} + +const contextStorage = new AsyncLocalStorage<{ readonly afs: BuildFsFragment }>() +async function getArtifactFs() { + const afs = contextStorage.getStore()?.afs + if (!afs) { + throw new Error(`No artifact fs available`) + } + return afs +} + +function runWithArtifactFs(afs: BuildFsFragment, fn: (...args: U) => T, ...args: U): T { + return contextStorage.run({ afs }, fn, ...args) +} + +interface GetAttrStep { + readonly type: 'get_attr' + readonly value: string +} + +interface IndexStep { + readonly type: 'index' + readonly value: string | { type: 'number'; value: number } +} + +type PathStep = GetAttrStep | IndexStep + +function getKey(step: PathStep) { + if (typeof step.value === 'object') { + return step.value.value + } + + return step.value +} + +function applyPath(target: any, steps: PathStep[]) { + if (steps.length === 0) { + return target + } + + return applyPath(target?.[getKey(steps[0])], steps.slice(1)) +} + +interface DataPointer { + readonly path: PathStep[] + readonly value: string +} + +interface EncodedPointers { + readonly prior?: DataPointer[] + readonly planned?: DataPointer[] +} + +interface ProviderConfig { + readonly outputDirectory: string + readonly workingDirectory: string +} + +interface BaseProviderRequest { + readonly type: string // This is the _custom_ resource type + readonly resourceName: string + readonly dependencies: string[] // Resource names + readonly operation: Operation + readonly priorInput?: any | null + readonly priorState?: any | null + readonly plannedState?: any | null + readonly providerConfig: ProviderConfig + readonly pointers?: EncodedPointers +} + +interface CreateRequest extends BaseProviderRequest { + readonly plannedState: any + readonly operation: 'create' +} + +interface DataRequest extends BaseProviderRequest { + readonly plannedState: any + readonly operation: 'data' +} + +interface ReadRequest extends BaseProviderRequest { + readonly priorInput: any + readonly priorState: any + readonly operation: 'read' +} + +interface UpdateRequest extends BaseProviderRequest { + readonly priorInput: any + readonly priorState: any + readonly plannedState: any + readonly operation: 'update' +} + +interface DeleteRequest extends BaseProviderRequest { + readonly priorInput: any + readonly priorState: any + readonly operation: 'delete' +} + +export type ProviderRequest = + | CreateRequest + | DataRequest + | ReadRequest + | UpdateRequest + | DeleteRequest + +function hydratePointers(req: T) { + const inputDeps = new Set() + if (!req.pointers) { + return { hydrated: req, inputDeps } + } + + function normalizePointers(type: 'input' | 'output', arr: DataPointer[] = []) { + return arr.filter(x => x.path[0].value === type).map(x => ({ ...x, path: x.path.slice(1) })) + } + + // This _mutates_ the object! + function hydrate(obj: any, arr: DataPointer[], isPlannedInput = false) { + for (const p of arr) { + const last = p.path.pop() + if (!last) { + continue + } + + const target = applyPath(obj, p.path) + if (!target) { + getLogger().warn(`Missing target [${p.path.map(x => x.value).join('.')}]: ${obj}`) + continue + } + + const key = getKey(last) + const hash = target[key] + if (p.value === '') { + target[key] = `${pointerPrefix}${hash}` + } else { + target[key] = createPointer(hash, p.value) + if (isPlannedInput) { + inputDeps.add(`${p.value}:${hash}`) + } + } + } + + return obj + } + + + const priorInputPointers = normalizePointers('input', req.pointers.prior) + const priorStatePointers = normalizePointers('output', req.pointers.prior) + const plannedStatePointers = normalizePointers('input', req.pointers.planned) + + const hydrated = { + ...req, + priorInput: hydrate(req.priorInput, priorInputPointers), + priorState: hydrate(req.priorState, priorStatePointers), + plannedState: hydrate(req.plannedState, plannedStatePointers, true), + } + + return { hydrated, inputDeps } +} + +export const resourceIdSymbol = Symbol.for('synapse.resourceId') + +function createProviderRoute(ctx: DeploymentContext, handlers: Handlers) { + interface ProviderResponse { + readonly state: T + readonly pointers?: Pointers + } + + async function runResourceRequestWithArtifactFs(payload: ProviderRequest, fn: (payload: ProviderRequest) => Promise): Promise> { + const { hydrated, inputDeps } = hydratePointers(payload) + const sfs = await createArtifactFs2(ctx, payload.resourceName, payload.dependencies, payload.operation) + + return runWithArtifactFs(sfs, fn, hydrated).then(async resp => { + // FIXME: `resp ?? null` is not entirely correct here. We have no way to serialize `undefined` + return ctx.processStore.saveResponse(payload.resourceName, Array.from(inputDeps), resp ?? null, payload.operation) + }).catch(e => { + const resourceId = `synapse_resource.${payload.resourceName}` + throw Object.assign(e, { [resourceIdSymbol]: resourceId }) + }) + } + + async function handleRequest(payload: ProviderRequest) { + switch (payload.type) { + case 'Asset': + assertNotData(payload.operation) + + return handlers.asset.handleAssetRequest({ + operation: payload.operation, + ...payload.priorInput, + ...payload.priorState, + ...payload.plannedState, + ...payload.providerConfig, + }) + + case 'Closure': + assertNotData(payload.operation) + + const location = payload.operation === 'read' + ? payload.priorInput.location + : payload.plannedState?.location + + return handlers.closure.handleClosureRequest({ + operation: payload.operation, + ...payload.priorInput, + ...payload.priorState, + ...payload.plannedState, + ...payload.providerConfig, + location, + }) + + case 'Custom': + assertNotData(payload.operation) + + return handlers.provider.handleProviderRequest({ + resource: payload.resourceName, + operation: payload.operation, + ...payload.priorInput, + ...payload.plannedState, + ...payload.providerConfig, + state: payload.priorState, + priorConfig: payload.priorInput, + }) + + case 'CustomData': + return handlers.provider.handleDataProviderRequest({ + resource: payload.resourceName, + operation: 'read', + ...payload.plannedState, + ...payload.providerConfig, + }) + + case 'ObjectData': { + const artifactFs = await getArtifactFs() + const data = payload.plannedState!.value + + // TODO: attach the symbol mapping as metadata so we can reverse the normalization + const normalized = isDeduped(data) ? normalizeSymbolIds(data) : data + const pointer = await artifactFs.writeData2(normalized) + + return { filePath: pointer } + } + + case 'Artifact': { + const pointer = payload.plannedState!.url + // TODO: remove this check? I think it's for backwards compat + if (!pointer.startsWith(pointerPrefix)) { + return { filePath: path.resolve(payload.providerConfig.workingDirectory, pointer) } + } + + const artifactFs = await getArtifactFs() + + return { filePath: await artifactFs.resolveArtifact(pointer) } + } + + case 'Test': + case 'TestSuite': { + assertNotData(payload.operation) + + switch (payload.operation) { + case 'create': + case 'update': + const { id, name, handler } = payload.plannedState + + return { id, name, handler } + case 'delete': + return {} + case 'read': + return payload.priorState + } + } + + case 'GetLogsCallback':{ + assertNotData(payload.operation) + + switch (payload.operation) { + case 'create': + case 'update': + const { resourceName, handler } = payload.plannedState + return { resourceName, handler } + case 'delete': + return {} + case 'read': + return payload.priorState + } + } + + // Effectively no-op resources + case 'ModuleExports': + return payload.plannedState ?? payload.priorState + + case apiRegistrationResourceType: + return getServiceRegistry().handleRequest(ctx, payload) + } + } + + const handlerRoute = new HttpRoute('/handle', async request => { + const payload = JSON.parse(await receiveData(request.request)) as ProviderRequest + const resp = await runResourceRequestWithArtifactFs(payload, handleRequest) + + return sendResponse(request.response, resp) + }) + + return [ + handlerRoute, + + // These routes are for backwards compat w/ "scaffolding" + // They can be removed after re-deploying (or destroying) all existing processes + new HttpRoute('/provider', async request => { + const payload = JSON.parse(await receiveData(request.request)) as any + const resp = await handlers.provider.handleProviderRequest(payload) + + return sendResponse(request.response, resp) + }), + + new HttpRoute('/assets', async request => { + const payload = JSON.parse(await receiveData(request.request)) as any + const resp = await handlers.asset.handleAssetRequest(payload) + + return sendResponse(request.response, resp) + }), + + new HttpRoute('/closure', async request => { + const payload = JSON.parse(await receiveData(request.request)) as any + const resp = await handlers.closure.handleClosureRequest(payload) + + return sendResponse(request.response, resp) + }), + + new HttpRoute('/hooks/{action}', async request => { + const { action } = request.params + const payload = JSON.parse(await receiveData(request.request)) as any + const loader = ctx.createModuleLoader() + const handler = await loader.loadModule(payload.handler) + const op = handler[action] + if (!op) { + throw new HttpError(`No operation found in handler: ${action}`, { statusCode: 404 }) + } + + if (action === 'beforeDestroy') { + const state = await op(payload.instance) + + return sendResponse(request.response, { state }) + } else if (action === 'afterCreate') { + await op(payload.instance, payload.state) + + return sendResponse(request.response, {}) + } else { + throw new HttpError(`Invalid operation: ${action}`, { statusCode: 404 }) + } + }) + ] +} + +async function getObject(val: string | undefined, isPointer = true) { + if (!val) { + return {} + } + + if (!isPointer) { + return JSON.parse(val) + } + + try { + const artifactFs = await getArtifactFs() + + return await artifactFs.readData2(val) + } catch (e) { + throw new Error(`Failed to resolve artifact: ${val}`, { cause: e }) + } +} + +function createClosureRoutes(ctx: DeploymentContext) { + interface BaseClosureRequest { + readonly options?: Record + readonly source: string // relative to the cwd + readonly location?: string + readonly workingDirectory: string + readonly outputDirectory: string + readonly operation: string + readonly captured: string // pointer + readonly globals?: string // pointer + } + + interface CreateClosureRequest extends BaseClosureRequest { + readonly operation: 'create' + } + + interface DeleteClosureRequest extends BaseClosureRequest { + readonly operation: 'delete' + readonly destination: string + } + + interface ReadClosureRequest extends BaseClosureRequest { + readonly operation: 'read' + readonly destination: string + readonly extname: string + readonly assets?: Record + } + + interface UpdateClosureRequest extends BaseClosureRequest { + readonly operation: 'update' + readonly destination: string + } + + type ClosureRequest = CreateClosureRequest | DeleteClosureRequest | ReadClosureRequest | UpdateClosureRequest + + async function handleClosureRequest(payload: ClosureRequest) { + async function deleteFile(payload: UpdateClosureRequest | DeleteClosureRequest) { + if (payload.destination.startsWith(pointerPrefix)) { + return + } + + const location = payload.destination + + try { + await ctx.fs.deleteFile(path.resolve(payload.workingDirectory, location)) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + getLogger().log('Failed to delete file', e) + } + } + + async function createClosure() { + const captured = await getObject(payload.captured) + const globals = await getObject(payload.globals) + const source = payload.source || `${payload.captured.slice(pointerPrefix.length)}.ts` + + const result = await bundleClosure( + ctx, + await getArtifactFs(), + source, + captured, + globals, + payload.workingDirectory, + payload.outputDirectory, + { + ...payload.options, + destination: payload.location, + } + ) + + const isResourceDef = result.location.startsWith(pointerPrefix) + const destination = isResourceDef ? result.location : path.relative(payload.workingDirectory, result.location) + + return { destination, extname: result.extname, assets: result.assets } + } + + switch (payload.operation) { + case 'create': + case 'update': + return await createClosure() + case 'delete': + await deleteFile(payload) + + return {} + case 'read': + if (payload.destination.startsWith(pointerPrefix)) { + return { destination: payload.destination, extname: payload.extname, assets: payload.assets } + } + + const location = path.resolve(payload.workingDirectory, payload.destination) + + if (!(await ctx.fs.fileExists(location))) { + getLogger().log('Re-creating missing closure', location) + + return await createClosure() + } + + return { destination: payload.destination, extname: payload.extname, assets: payload.assets } + + default: + throw new Error(`Unknown operation: ${(payload as any).operation}`) + } + } + + return { handleClosureRequest } +} + +enum AssetType { + FILE = 0, + DIRECTORY = 1, + ARCHIVE = 2 +} + +function createAssetRoute(ctx: DeploymentContext, terraformWorkingDirectory: string) { + interface BaseAssetRequest { + readonly path: string + readonly extname?: string + readonly type?: AssetType + readonly extraFiles?: Record + readonly workingDirectory: string + readonly outputDirectory: string + readonly operation: string + } + + interface CreateAssetRequest extends BaseAssetRequest { + readonly operation: 'create' + } + + interface DeleteAssetRequest extends BaseAssetRequest { + readonly operation: 'delete' + readonly filePath: string + } + + interface ReadAssetRequest extends BaseAssetRequest { + readonly operation: 'read' + readonly filePath: string + readonly sourceHash?: string + } + + interface UpdateAssetRequest extends BaseAssetRequest { + readonly operation: 'update' + } + + type AssetRequest = CreateAssetRequest | DeleteAssetRequest | ReadAssetRequest | UpdateAssetRequest + + async function zip(dir: string, target: string, dest: string, targetIsDir = false, extraFiles?: Record) { + if (ctx.createZip) { + const files: Record = targetIsDir + ? await readDirRecursive(ctx.fs, dir) + : { [target]: path.resolve(dir, target) } + + if (extraFiles) { + for (const [k, v] of Object.entries(extraFiles)) { + const name = k.startsWith('file:./') ? k.slice('file:./'.length) : k + const location = v.startsWith('pointer:') ? resolveArtifact(dir, v) : path.resolve(terraformWorkingDirectory, v) + files[name] = await location + } + } + + try { + return await ctx.createZip(files, dest) + } catch (e) { + getLogger().debug(`Failed to use built-in zip command`, e) + } + } + + getLogger().debug(`Running "zip" on file "${target}" from`, dir) + + // TODO: use `tar` if `zip` isn't available + const c = child_process.spawn('zip', ['-r', dest, `./${target}`], { + cwd: dir, + }) + + // FIXME: use common impl. for spawning processes + function logZip(d: Buffer) { + getLogger().log(`child process [zip]: ${d.toString('utf-8').trimEnd()}`) + } + + c.stdout?.on('data', logZip) + c.stderr?.on('data', logZip) + + return new Promise((resolve, reject) => { + c.on('error', reject) + c.on('exit', code => { + if (code !== 0) { + reject(new Error(`Non-zero exit code: ${code}`)) + } else { + resolve() + } + }) + }) + } + + async function computeHash(fileName: string, encoding: BinaryToTextEncoding = 'base64url') { + return createHash('sha256').update(await fs.readFile(fileName)).digest(encoding) + } + + async function computeHashDir(dir: string): Promise { + const hash = createHash('sha256') + for (const f of await fs.readdir(dir, { withFileTypes: true })) { + const absPath = path.resolve(dir, f.name) + // TODO: follow sym links? + if (f.isFile()) { + hash.update(await fs.readFile(absPath)) + } else if (f.isDirectory()) { + hash.update(await computeHashDir(absPath)) + } + } + + return hash.digest('base64url') + } + + // XXX: this is used for Lambdas to ensure a static handler name + // Otherwise we get this error from the AWS TF provider: `handler and runtime must be set when PackageType is Zip` + async function resolveArtifact(dir: string, p: string, extname = '') { + const resolved = path.resolve(dir, `handler${extname}`) + const afs = await getArtifactFs() + + return afs.resolveArtifact(p, { filePath: resolved }) + } + + async function handleAssetRequest(payload: AssetRequest) { + async function createAsset() { + const tmpDir = path.resolve(terraformWorkingDirectory, 'archives', 'tmp', randomUUID()) + + const target = payload.path.startsWith(pointerPrefix) + ? await resolveArtifact(tmpDir, payload.path, payload.extname) + : path.resolve(payload.workingDirectory, payload.path) + + if (payload.type === AssetType.ARCHIVE) { + const isDir = !payload.path.startsWith(pointerPrefix) && (await fs.stat(target)).isDirectory() + const sourceHash = await (isDir ? computeHashDir(target) : computeHash(target)) + const outFile = `${sourceHash}.zip` + const relPath = path.join('archives', outFile) + const dest = path.resolve(terraformWorkingDirectory, relPath) + await ensureDir(dest) + + if (isDir) { + await zip(target, '.', dest, true, payload.extraFiles) + } else { + await zip(path.dirname(target), path.basename(target), dest, false, payload.extraFiles) + } + + await fs.rm(tmpDir, { force: true, recursive: true }) + + return { + sourceHash, + filePath: relPath, + } + } + + return { + filePath: target, + } + } + + if (payload.operation === 'create' || payload.operation === 'update') { + return createAsset() + } else if (payload.operation === 'delete') { + const dest = path.resolve(terraformWorkingDirectory, payload.filePath) + + try { + await fs.rm(dest) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } + + return {} + } else if (payload.operation === 'read') { + const dest = path.resolve(terraformWorkingDirectory, payload.filePath) + if (!payload.filePath || !(await ctx.fs.fileExists(dest))) { + return createAsset() + } + + return { filePath: payload.filePath, sourceHash: payload.sourceHash } + } + } + + return { handleAssetRequest } +} + +interface Status { + workingDirectory: string + startTime?: Date +} + +function createStatusRoute(status: Status) { + return new HttpRoute('/status', async request => { + const startTime = status.startTime?.getTime() + const uptime = startTime ? Date.now() - startTime : -1 + + return sendResponse(request.response, { workingDirectory: status.workingDirectory, uptime }) + }) +} + +interface Handlers { + closure: ReturnType + provider: ReturnType + asset: ReturnType +} + +export async function startService( + ctx: DeploymentContext, + rootDir: string, + port?: number, + terraformWorkingDirectory = rootDir +) { + const stateDir = path.join(rootDir, '.state') + const stateHandler = createStateRoute(stateDir) + const status = { workingDirectory: rootDir } as Status + const server = HttpServer.fromRoutes( + ...stateHandler, + createStatusRoute(status), + ...createProviderRoute(ctx, { + closure: createClosureRoutes(ctx), + provider: createProviderRoutes(ctx), + asset: createAssetRoute(ctx, terraformWorkingDirectory) + }), + ) + + const boundPort = await server.start(port) + status.startTime = new Date() + getLogger().log(`Listening to port ${boundPort} at`, rootDir) + + return { + port: boundPort, + takeError: (requestId: string) => server.takeError(requestId), + dispose: () => server.close(), + } +} + diff --git a/src/deploy/session.ts b/src/deploy/session.ts new file mode 100644 index 0000000..465bc89 --- /dev/null +++ b/src/deploy/session.ts @@ -0,0 +1,481 @@ +import * as path from 'node:path' +import { getLogger } from '..' +import { createMountedFs, getPublished, getDeploymentFs, getDataRepository, getDeploymentStore, readState, toFsFromIndex, createTempMountedFs, getFsFromHash, toFsFromHash, getProgramFs, getProgramHash, DataRepository } from '../artifacts' +import { getAuth } from '../auth' +import { getBackendClient } from '../backendClient' +import { isDataPointer } from '../build-fs/pointers' +import { getBuildTargetOrThrow, getFs } from '../execution' +import { createPackageService, importMapToManifest } from '../pm/packages' +import { createMergedView, getSelfDir } from '../pm/publish' +import { ImportMap, SourceInfo } from '../runtime/importMaps' +import { createContext, createSourceMapParser, createModuleLoader, BasicDataRepository } from '../runtime/loader' +import { resolveValue } from '../runtime/modules/serdes' +import { createBasicDataRepo, createModuleResolverForBundling } from '../runtime/rootLoader' +import { createCodeCache } from '../runtime/utils' +import { createTemplateService } from '../templates' +import { wrapWithProxy, memoize, getHash, createRwMutex, throwIfNotFileNotFoundError } from '../utils' +import { BuildTarget, getOrCreateDeployment, getDeploymentBuildDirectory, getV8CacheDirectory } from '../workspaces' +import { DeployOptions, SessionContext, createStatePersister, createZip, getTerraformPath, mapResource, parsePlan, startTerraformSession } from './deployment' +import { TfJson } from '../runtime/modules/terraform' +import { TfState } from './state' +import { printLine } from '../cli/ui' +import { formatWithOptions } from 'node:util' +import { getDeployables, getEntrypointsFile } from '../compiler/programBuilder' +import { ModuleLoader } from './server' +import { Fs, SyncFs } from '../system' +import { ModuleResolver } from '../runtime/resolver' +import { getServiceRegistry } from './registry' +import { getCurrentEnvFilePath, parseEnvFile } from '../runtime/env' + +export async function loadBuildState(bt: BuildTarget, repo = getDataRepository()) { + const mergedFs = await createMergedView(bt.programId, bt.deploymentId) + const mountedFs = createTempMountedFs(mergedFs, bt.workingDirectory) + const resolver = createModuleResolverForBundling(mountedFs, bt.workingDirectory) + const pkgService = await createPackageService(resolver, repo) + const { stores } = await pkgService.loadIndex() // 5ms on simple hello world no infra + mountedFs.addMounts(stores) + + await setupPublished() + + return { + repo, + mountedFs: mountedFs as Fs & SyncFs, + resolver: resolver as ModuleResolver, + registerPointerDependencies: pkgService.registerPointerDependencies, + } + + async function setupPublished() { + if (!bt.deploymentId) { + return + } + + const procFs = getDeploymentFs(bt.deploymentId, bt.programId) + const published = await getPublished(procFs) + if (!published) { + return + } + + const deployables = (await getEntrypointsFile(getProgramFs(bt.programId)))?.deployables + const programDeployables = new Set(Object.values(deployables ?? {}).map(f => path.relative(bt.workingDirectory, f))) + + const importMap: ImportMap = {} + for (const [k, v] of Object.entries(published)) { + if (!programDeployables.has(k)) continue + + const m = await pkgService.getPublishedMappings2(k, v) + if (m) { + importMap[m[0]] = m[1] + } + } + + resolver.registerMapping(importMap, bt.workingDirectory) + } +} + +export async function getModuleLoader(wrapConsole = true): Promise { + const bt = getBuildTargetOrThrow() + const repo = getDataRepository() + const auth = getAuth() + const fs = getFs() + const backendClient = getBackendClient() + + const { mountedFs, resolver, registerPointerDependencies } = await loadBuildState(bt) + + function createConsoleWrap() { + const printToConsole = (...args: any[]) => printLine(formatWithOptions({ colors: process.stdout.isTTY }, ...args)) + const logMethods = { + log: printToConsole, + warn: printToConsole, + error: printToConsole, + debug: printToConsole, + // TODO: this is wrong, we don't emit a trace + trace: printToConsole, + } + + return wrapWithProxy(globalThis.console, logMethods) + } + + + const loaderContext = createContext(bt, backendClient, auth, wrapConsole ? createConsoleWrap() : undefined) + + const getSourceMapParser = memoize(async () => { + const selfDir = await getSelfDir() + const sourceMapParser = createSourceMapParser(mountedFs, resolver, bt.workingDirectory, selfDir ? [selfDir] : undefined) + loaderContext.registerSourceMapParser(sourceMapParser) + + return sourceMapParser + }) + + const sourceMapParser = await getSourceMapParser() + + function createModuleLoader2(): ReturnType { + const dataDir = repo.getDataDir() + const codeCache = createCodeCache(fs, getV8CacheDirectory()) + + const loader = createModuleLoader( + mountedFs, + dataDir, + resolver, + { + sourceMapParser, + workingDirectory: bt.workingDirectory, + codeCache, + deserializer: resolveValue, + dataRepository: createBasicDataRepo(repo), + } + ) + + async function loadModule(id: string, origin?: string) { + if (isDataPointer(id)) { + await registerPointerDependencies(id) + } + + return loader(origin, loaderContext.ctx)(id) + } + + return { + loadModule, + registerMapping: resolver.registerMapping, + runWithContext: async (namedContexts: Record, fn: () => Promise | T) => { + return loaderContext.runWithNamedContexts(namedContexts, fn) + }, + } + } + + return createModuleLoader2() +} + +// TODO: this isn't clean +async function maybeLoadEnvironmentVariables(fs: Pick) { + const filePath = getCurrentEnvFilePath() + getLogger().debug(`Trying to load environment variables from "${filePath}"`) + + const text = await fs.readFile(filePath, 'utf-8').catch(throwIfNotFileNotFoundError) + if (!text) { + return + } + + const vars = parseEnvFile(text) + for (const [k, v] of Object.entries(vars)) { + process.env[k] = v + } + + getLogger().debug(`Loaded environment variables: ${Object.keys(vars)}`) +} + +export async function createSessionContext(programHash?: string): Promise { + const bt = getBuildTargetOrThrow() + const deploymentId = await getOrCreateDeployment() + const repo = getDataRepository() + + const resolvedProgramHash = programHash ?? (await repo.getHead(bt.programId))!.storeHash + const tmpMountedFs = createTempMountedFs((await repo.getBuildFs(resolvedProgramHash)).index, bt.workingDirectory) + + const fs = tmpMountedFs + const terraformPath = await getTerraformPath() + const templateService = createTemplateService(fs, programHash) + + const workingDirectory = bt.workingDirectory + + function createModuleResolver2() { + return createModuleResolverForBundling(fs, workingDirectory) + } + + const resolver = createModuleResolver2() + const auth = getAuth() + const backendClient = getBackendClient() + + function createConsoleWrap() { + const logEvent = getLogger().emitDeployLogEvent + const logMethods = { + log: (...args: any[]) => logEvent({ level: 'info', args, resource: loaderContext.getContext('resource') }), + warn: (...args: any[]) => logEvent({ level: 'warn', args, resource: loaderContext.getContext('resource') }), + error: (...args: any[]) => logEvent({ level: 'error', args, resource: loaderContext.getContext('resource') }), + debug: (...args: any[]) => logEvent({ level: 'debug', args, resource: loaderContext.getContext('resource') }), + trace: (...args: any[]) => logEvent({ level: 'trace', args, resource: loaderContext.getContext('resource') }), + } + + return wrapWithProxy(globalThis.console, logMethods) + } + + const consoleWrap = createConsoleWrap() + const loaderContext = createContext({ deploymentId: deploymentId, programId: bt.programId }, backendClient, auth, consoleWrap) + + const getSourceMapParser = memoize(async () => { + const selfDir = await getSelfDir() + const sourceMapParser = createSourceMapParser(fs, resolver, workingDirectory, selfDir ? [selfDir] : undefined) + loaderContext.registerSourceMapParser(sourceMapParser) + + return sourceMapParser + }) + + // XXX: only used for `internal/apps/github` atm + await maybeLoadEnvironmentVariables(fs) + + const env = { + SYNAPSE_ENV: bt.environmentName, + ...(await templateService.getSecretBindings()) + } + + const sourceMapParser = await getSourceMapParser() + + const pkgService = await createPackageService(resolver, repo, programHash ? await toFsFromHash(programHash) : undefined) + const { stores, importMap } = await pkgService.loadIndex() + const mountedFs = await createMountedFs(deploymentId, workingDirectory, stores) + tmpMountedFs.addMounts(stores) + + function createModuleLoader2(): ReturnType { + const dataDir = repo.getDataDir() + const codeCache = createCodeCache(fs, getV8CacheDirectory()) + + const loader = createModuleLoader( + mountedFs, + dataDir, + resolver, + { + env, + sourceMapParser, + workingDirectory, + codeCache, + deserializer: resolveValue, + dataRepository: createBasicDataRepo(repo), + } + ) + + async function loadModule(id: string, origin?: string) { + if (isDataPointer(id)) { + await pkgService.registerPointerDependencies(id) + } + + return loader(origin, loaderContext.ctx)(id) + } + + return { + loadModule, + registerMapping: resolver.registerMapping, + runWithContext: async (namedContexts: Record, fn: () => Promise | T) => { + return loaderContext.runWithNamedContexts(namedContexts, fn) + }, + } + } + + const packageManifest = importMap ? importMapToManifest(importMap) : undefined + + return { + dataDir: repo.getDataDir(), + packageManifest: packageManifest ?? { roots: {}, dependencies: {}, packages: {} }, + packageService: pkgService, + buildTarget: { ...bt, deploymentId: deploymentId, programHash }, + templateService, + terraformPath, + fs: mountedFs, + processStore: getDeploymentStore(deploymentId, repo), + backendClient, + createModuleResolver: () => resolver, + createModuleLoader: createModuleLoader2, + createZip, + } +} + +const sessions = new Map>() + +export async function createSession(ctx: SessionContext, opt?: DeployOptions) { + // These options need to be added when starting the session. They have no effect otherwise. + const args: string[] = ['-auto-approve', '-refresh=false'] + if (opt?.parallelism) { + args.push(`-parallelism=${opt.parallelism}`) + } + + const programHash = ctx.buildTarget.programHash ?? await getProgramHash() + if (!programHash) { + throw new Error(`Missing program hash from build target: ${JSON.stringify(ctx.buildTarget, undefined, 4)}`) + } + + const session = await startTerraformSession(ctx, args, opt) + const moduleLoader = ctx.createModuleLoader() + + let state: TfState | undefined + let persister: ReturnType | undefined + async function loadState(noSave = opt?.noSave ?? false) { + await persister?.dispose() + persister = undefined + + state = await readState() + if (state) { + const stateDest = path.resolve(getDeploymentBuildDirectory(ctx.buildTarget), 'state.json') + await getFs().writeFile(stateDest, JSON.stringify(state)) + try { + await session.setState(stateDest) + } finally { + await getFs().deleteFile(stateDest) + } + + //await getServiceRegistry().loadFromState(ctx, state) + } + + if (!noSave) { + ensurePersister() + } + + return state + } + + // await loadState() + + let templateHash: string + let optionsHash: string + async function shouldRun(opt?: DeployOptions) { + const currentTemplateHash = await ctx.templateService.getTemplateHash() + + const currentOptionsHash = getHash(JSON.stringify({ ...opt }), 'base64url') + if ((optionsHash && currentOptionsHash === optionsHash) && (templateHash && templateHash === currentTemplateHash)) { + return false + } + + if (templateHash && templateHash !== currentTemplateHash) { + await session.reloadConfig() + // await loadState() + } + + templateHash = currentTemplateHash + optionsHash = currentOptionsHash + + return true + } + + const loadStateOnce = memoize(loadState) + + function ensurePersister() { + persister ??= createStatePersister(state, programHash!) + } + + async function finalizeState() { + const newState = await session.getState() + newState.lineage = persister?.getLineage() ?? newState.lineage + newState.serial = persister?.getNextSerial() ?? (newState.serial + 1) + await persister?.dispose() + persister = undefined + + return state = newState + } + + async function apply(opt?: DeployOptions) { + await loadStateOnce() + + let error: Error | undefined + if (await shouldRun(opt)) { + ensurePersister() + error = await session.apply(opt).catch(e => e) + + return { + error, + state: await finalizeState(), + } + } else { + if (!state) { + throw new Error(`Missing state`) + } + + getLogger().log('No changes detected, skipping apply') + // XXX: emit a stub plan to appease UI + getLogger().emitPlanEvent({ plan: {} }) + + return { state } + } + } + + async function destroy(opt?: DeployOptions) { + await loadStateOnce() + ensurePersister() + const error = await session.destroy(opt).catch(e => e) + + return { + error, + state: await finalizeState(), + } + } + + async function plan(opt?: DeployOptions) { + const state = await loadStateOnce() + const p = await session.plan(opt) + const resources: Record = {} + if (state?.resources) { + for (const r of state?.resources) { + const key = `${r.type}.${r.name}` + resources[key] = mapResource(r)?.state + } + } + + return parsePlan(p, resources) + } + + + async function _dispose() { + getLogger().log('Shutting down session', ctx.buildTarget.deploymentId!) + sessions.delete(ctx.buildTarget.deploymentId!) + await session.dispose() + await persister?.dispose() + } + + const dispose = memoize(_dispose) + + const lock = createRwMutex() + function wrapWithLock(fn: (...args: U) => Promise | T): (...args: U) => Promise { + return async (...args) => { + const l = await lock.lockWrite() + + try { + return await fn.apply(undefined, args) + } finally { + l.dispose() + } + } + } + + async function setTemplate(template: TfJson) { + await ctx.templateService.setTemplate(template) + await session.reloadConfig() + } + + async function getState() { + await loadStateOnce() + + return state + } + + return { + plan, + apply: wrapWithLock(apply), + destroy: wrapWithLock(destroy), + dispose: wrapWithLock(dispose), + getState, + getRefs: session.getRefs, + setTemplate, + templateService: ctx.templateService, + moduleLoader, + } +} + +export function getSession(deploymentId: string, ctx?: SessionContext, opt?: DeployOptions) { + if (sessions.has(deploymentId)) { + return sessions.get(deploymentId)! + } + + async function createWithCtx() { + return createSession(ctx ?? await createSessionContext(), opt) + } + + const session = createWithCtx() + sessions.set(deploymentId, session) + + return session +} + +export async function shutdownSessions() { + for (const [key, session] of sessions.entries()) { + const s = await session + await s.dispose() + sessions.delete(key) + } +} + diff --git a/src/deploy/state.ts b/src/deploy/state.ts new file mode 100644 index 0000000..5aec1db --- /dev/null +++ b/src/deploy/state.ts @@ -0,0 +1,47 @@ +// TODO: move relevant util functions to this file + +export interface TfResourceInstance { + status?: 'tainted' + schema_version: number + attributes: Record + private?: string + create_before_destroy?: boolean + dependencies?: string[] + sensitive_attributes?: { + type: 'get_attr' + value: string + }[] +} + +export interface TfResourceOld { + type: string + name: string + provider: string + instances: TfResourceInstance[] +} + +export interface TfResource { + type: string + name: string + provider: string + state: TfResourceInstance +} + +export interface TfStateOld { + version: number + serial: number + lineage: string + resources: TfResourceOld[] +} + +export interface TfState { + version: number + serial: number + lineage: string + resources: TfResource[] +} + +export interface AnnotatedTfState extends TfState { + serial: number + lineage: string +} diff --git a/src/events.ts b/src/events.ts new file mode 100644 index 0000000..2a362be --- /dev/null +++ b/src/events.ts @@ -0,0 +1,58 @@ +// FIXME: `default` imports don't work correctly for cjs bundles +// import EventEmitter from 'node:events' +import { EventEmitter } from 'node:events' +export { EventEmitter } + +export interface Disposable { + dispose: () => void +} + +export interface Event { + fire(...args: T): void + on(listener: (...args: T) => void): Disposable +} + +export function createEventEmitter() { + return new EventEmitter() +} + +const listenerSymbol = Symbol('listener') + +interface ListenerEvent { + readonly eventName: string | symbol + readonly mode: 'added' | 'removed' +} + +export function addMetaListener(emitter: EventEmitter, listener: (ev: ListenerEvent) => void) { + emitter.on(listenerSymbol, listener) + + return { dispose: () => emitter.removeListener(listenerSymbol, listener) } +} + +export type EventEmitter2 = (listener: (ev: T) => void) => Disposable + +export function createEvent(emitter: EventEmitter, type: U): Event { + return { + fire: (...args) => emitter.emit(type, ...args), + on: listener => { + emitter.on(type, listener as any) + emitter.emit(listenerSymbol, { eventName: type, mode: 'added' }) + + function dispose() { + emitter.removeListener(type, listener as any) + emitter.emit(listenerSymbol, { eventName: type, mode: 'removed' }) + } + + return { dispose } + }, + } +} + +export function once(event: Event, fn: (...args: T) => void): Disposable { + const d = event.on((...args) => { + d.dispose() + fn(...args) + }) + + return d +} diff --git a/src/execution.ts b/src/execution.ts new file mode 100644 index 0000000..ee1a4f9 --- /dev/null +++ b/src/execution.ts @@ -0,0 +1,106 @@ +import { randomUUID } from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' +import { Fs, SyncFs, createLocalFs } from './system' +import { BuildTarget } from './workspaces' + +interface ExecutionContext { + readonly id: string + readonly fs: Fs & SyncFs + readonly selfPath?: string + readonly selfBuildType?: 'snapshot' | 'sea' + readonly buildTarget?: BuildTarget + readonly abortSignal?: AbortSignal +} + +// Looks interesting +// https://github.com/nodejs/node/issues/46265 +// Removing most of the `async_hooks` code would be great +// The most useful thing is by far `AsyncLocalStorage` + +const storage = new AsyncLocalStorage() + +function getContextOrThrow() { + const ctx = storage.getStore() + if (!ctx) { + throw new Error(`Not within an execution context`) + } + + return ctx +} + +let defaultContext: Partial +export function setContext(ctx: Partial) { + defaultContext = ctx +} + +export function runWithContext(ctx: Partial, fn: () => T): T { + const previousStore = storage.getStore() + const id = ctx.id ?? previousStore?.id ?? randomUUID() + const fs = ctx.fs ?? previousStore?.fs ?? createLocalFs() + + return storage.run({ ...previousStore, ...ctx, id, fs }, fn) +} + +export function getFs() { + return storage.getStore()?.fs ?? createLocalFs() +} + +export function getExecutionId() { + return defaultContext?.id ?? getContextOrThrow().id +} + +export function getBuildTarget() { + return storage.getStore()?.buildTarget +} + +export function isInContext() { + return storage.getStore() !== undefined +} + +export function getBuildTargetOrThrow() { + const bt = getBuildTarget() + if (!bt) { + throw new Error(`No build target found`) + } + + return bt +} + +export function throwIfCancelled() { + storage.getStore()?.abortSignal?.throwIfAborted() +} + +export function isCancelled() { + return !!storage.getStore()?.abortSignal?.aborted +} + +export function isSelfSea() { + if (defaultContext?.selfBuildType === 'sea') { + return true + } + + return storage.getStore()?.selfBuildType === 'sea' +} + +export function getSelfPath() { + if (defaultContext?.selfPath) { + return defaultContext?.selfPath + } + + return storage.getStore()?.selfPath +} + +export function getSelfPathOrThrow() { + if (defaultContext?.selfPath) { + return defaultContext?.selfPath + } + + const p = getContextOrThrow().selfPath + if (!p) { + throw new Error('Missing self path') + } + + return p +} + +export class CancelError extends Error {} diff --git a/src/git.ts b/src/git.ts new file mode 100644 index 0000000..37e72d5 --- /dev/null +++ b/src/git.ts @@ -0,0 +1,216 @@ +import * as path from 'node:path' +import * as child_process from 'node:child_process' +import { mkdir } from 'node:fs/promises' +import { runCommand } from './utils/process' +import { getFs } from './execution' +import { getGitDirectory } from './workspaces' +import { ensureDir } from './utils' + +async function runGit(cwd: string, args: string[]) { + return await runCommand('git', args, { cwd }) +} + +export interface Remote { + readonly name: string + readonly fetchUrl: string + readonly pushUrl: string + readonly headBranch: string +} + +async function isInWorkTree(dir: string) { + const output = await runGit(dir, ['rev-parse', '--is-inside-work-tree']) + + return output.trim() === 'true' +} + +// We simply check for a `.git` directory +export async function findRepositoryDir(dir: string): Promise { + const gitDir = path.resolve(dir, '.git') + if (await getFs().fileExists(gitDir)) { + return dir + } + + const parentDir = path.dirname(dir) + if (parentDir !== dir) { + return findRepositoryDir(parentDir) + } +} + +async function getUrls(dir: string, remote: string) { + // TODO: use `git ls-remote --get-url ${remote}` too + const fetchPromise = runGit(dir, ['config', '--get', `remote.${remote}.url`]) + const pushPromise = runGit(dir, ['config', '--get', `remote.${remote}.pushurl`]).catch(e => { + if (e.code === 1) { + return fetchPromise + } + + throw e + }) + + const [fetch, push] = await Promise.all([fetchPromise, pushPromise]) + + return { + fetchUrl: fetch.trim(), + pushUrl: push.trim(), + } +} + +async function listBranches(dir: string, url: string) { + function parseBranch(line: string) { + const match = line.match(/([^\s])+\s+refs\/heads\/([^\s]+)/) + if (!match) { + throw new Error(`Unable to parse branch from line: ${line}`) + } + + return { name: match[2], commit: match[1] } + } + + const lsRemote = await runGit(dir, ['ls-remote', '--heads', url]) + + return lsRemote.trim().split('\n').map(parseBranch) +} + +async function parseRemoteSlow(dir: string, remote: string): Promise { + const { fetchUrl, pushUrl } = await getUrls(dir, remote) + const lsRemote = await runGit(dir, ['ls-remote', '--symref', fetchUrl]) + const match = lsRemote.match(/ref: refs\/heads\/([^\s]+)\s+HEAD/) + if (!match) { + throw new Error(`No HEAD ref found: ${lsRemote}`) + } + + return { + pushUrl, + fetchUrl, + name: remote, + headBranch: match[1], + } +} + +async function parseRemoteNoBranch(dir: string, remote: string): Promise> { + const { fetchUrl, pushUrl } = await getUrls(dir, remote) + + return { + name: remote, + pushUrl, + fetchUrl, + } +} + +export async function listRemotes(dir = process.cwd()) { + const result = await runGit(dir, ['remote', 'show']) + const remoteNames = result.trim().split('\n').filter(x => !!x) + const remotes = remoteNames.map(n => parseRemoteNoBranch(dir, n)) + + return await Promise.all(remotes) +} + +export async function getCurrentBranch(dir = process.cwd()) { + // fatal: not a git repository (or any of the parent directories): .git + const branch = await runGit(dir, ['branch', '--show-current']).catch(e => { + if (e.code !== 128) { + throw e + } + }) + + return branch ? branch.trim() : undefined +} + +export async function getLatestCommit(remote: string, branch = 'main', dir = process.cwd()) { + await runGit(dir, ['fetch', remote, branch]) + const hash = await runGit(dir, ['rev-parse', 'FETCH_HEAD']) + + return hash.trim() +} + +export function getCurrentBranchSync(dir = process.cwd()) { + const res = child_process.execFileSync('git', ['branch', '--show-current'], { cwd: dir, encoding: 'utf-8' }) + + return res.trim() +} + +export async function openRemote(remote: string) { + const dest = path.resolve(getGitDirectory(), 'remotes', remote) + await mkdir(dest, { recursive: true }) + + const cloneResult = await runCommand( + 'git', + ['clone', '--depth', '1', '--no-checkout', '--no-tags', '--filter=blob:none', remote, dest] + ) + + const treeResult = await runCommand( + 'git', + ['ls-tree', '-r', '-z', 'HEAD'], + { cwd: dest } + ) + + let isDisposed = false + + async function readFile(type: string, hash: string, encoding?: BufferEncoding): Promise + async function readFile(type: string, hash: string, encoding: BufferEncoding): Promise + async function readFile(type: string, hash: string, encoding?: BufferEncoding) { + return runCommand('git', ['cat-file', type, hash], { + cwd: dest, + encoding: encoding ?? 'none', + }) as Promise + } + + const files = treeResult + .toString() + .slice(0, -1) + .split(/\0/) + .map(s => s.split(/\s/)) + .map(([mode, type, hash, name]) => ({ + name: name!, + read: async () => { + if (isDisposed) { + throw new Error(`Cannot read file "${name}" after the remote has been disposed`) + } + + return readFile(type!, hash!) + } + })) + + async function dispose() { + isDisposed = true + await getFs().deleteFile(dest) + } + + return { + files, + dispose, + } +} + +export async function fetchOriginHead(dir: string, commitish: string) { + await runGit(dir, ['fetch', 'origin', commitish]) + await runGit(dir, ['reset', '--hard', 'FETCH_HEAD']) +} + +export async function fetchRepo(dir: string, source: string, commitish: string) { + await ensureDir(dir) + await runGit(dir, ['init']), + await runGit(dir, ['remote', 'add', 'origin', source]) + await fetchOriginHead(dir, commitish) +} + +// async function cloneRepo(procId: string, request: CreateProcessRequest, cwd?: string) { +// const dest = cwd ? path.resolve(cwd, procId) : procId +// const cloneArgs = [ +// 'clone', +// '--depth', +// '1', +// '--single-branch', +// '--branch', +// request.branch, +// getCloneUrl(request.owner, request.repo, request.accessToken), +// dest +// ] + +// await runCommand('git', cloneArgs, { stdio: 'inherit' }).promise + +// return dest +// } + +// const getCloneUrl = (owner: string, repo: string, token: string) => +// `https://x-access-token:${token}@github.com/${owner}/${repo}.git` + diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..6f5c9b3 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,2863 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { CompilerOptions, CompilerHost, synth, CompiledSource, readPointersFile, readSources } from './compiler/host' +import { FailedTestEvent, TestEvent, runTask } from './logging' +import { BoundTerraformSession, DeployOptions, SessionContext, SessionError, createStatePersister, createZipFromDir, getChangeType, getDiff, getTerraformPath, isTriggeredReplaced, parsePlan, startStatelessTerraformSession, startTerraformSession } from './deploy/deployment' +import { LocalWorkspace, getV8CacheDirectory, initProject, getLinkedPackagesDirectory, Program, getRootDirectory, getDeploymentBuildDirectory, getTargetDeploymentIdOrThrow, getOrCreateDeployment, getWorkingDir } from './workspaces' +import { createLocalFs } from './system' +import { AmbientDeclarationFileResult, Mutable, acquireFsLock, createHasher, createRwMutex, getCiType, isNonNullable, isWindows, keyedMemoize, makeExecutable, memoize, printNodes, replaceWithTilde, showArtifact, throwIfNotFileNotFoundError, toAmbientDeclarationFile, wrapWithProxy } from './utils' +import { SymbolGraph, SymbolNode, createMergedGraph, createSymbolGraph, createSymbolGraphFromTemplate, detectRefactors, normalizeConfigs, renderSymbol, renderSymbolLocation } from './refactoring' +import { SourceMapHost } from './static-solver/utils' +import { getLogger } from './logging' +import { createContext, createModuleLoader, createSourceMapParser } from './runtime/loader' +import { BuildFsIndex, CompiledChunk, TemplateWithHashes, checkBlock, commitProgram, createArtifactFs, createBuildFsFragment, createMountedFs, getDataRepository, getFsFromHash, getInstallation, getMoved, getPreviousDeploymentProgramHash, getDeploymentFs, getProgramFs, getProgramHash, getResourceProgramHashes, listCommits, maybeRestoreTemplate, printBlockInfo, putState, readResourceState, readState, saveMoved, shutdownRepos, syncRemote, toFs, toFsFromHash, writeTemplate } from './artifacts' +import { PackageService, createPackageService, maybeDownloadPackages, showManifest, downloadAndUpdatePackage, verifyInstall, downloadAndInstall, listInstall, resolveDepsGreedy, printTree } from './pm/packages' +import { createTestRunner, listTestSuites, listTests } from './testing' +import { ReplOptions, enterRepl, createReplServer, prepareReplWithSymbols, getSymbolDataResourceId } from './repl' +import { createTemplateService, getHash, parseModuleName } from './templates' +import { createImportMap, createModuleResolver } from './runtime/resolver' +import { createAuth, getAuth } from './auth' +import { generateOpenApiV3, generateStripeWebhooks } from './codegen/schemas' +import { addImplicitPackages, createMergedView, createNpmLikeCommandRunner, dumpPackage, emitPackageDist, getPkgExecutables, getProjectOverridesMapping, installToUserPath, linkPackage } from './pm/publish' +import { ResolvedProgramConfig, getResolvedTsConfig, resolveProgramConfig } from './compiler/config' +import { createProgramBuilder, getDeployables, getEntrypointsFile, getExecutables } from './compiler/programBuilder' +import { loadCpuProfile } from './perf/profiles' +import { colorize, createTreeView, printJson, printLine, print, getDisplay, bold, RenderableError, dim } from './cli/ui' +import { createDeployView, extractSymbolInfoFromPlan, groupSymbolInfoByFile, printSymbolTable, renderBetterSymbolName, renderSummary, renderSymbolWithState } from './cli/views/deploy' +import { TfJson } from './runtime/modules/terraform' +import { glob } from './utils/glob' +import { createMinimalLoader } from './runtime/rootLoader' +import { getBackendClient } from './backendClient' +import { createCodeCache } from './runtime/utils' +import { getBuildTarget, getBuildTargetOrThrow, getFs, getSelfPathOrThrow, isCancelled, isSelfSea, throwIfCancelled } from './execution' +import * as secrets from './services/secrets' +import * as workspaces from './workspaces' +import { createTestView } from './cli/views/test' +import { clearIncrementalCache, createIncrementalHost, getAllDependencies, getFileHasher } from './compiler/incremental' +import { getMostRecentLogFile, getSortedLogs, listLogFiles } from './cli/logger' +import { ResolvedPackage, getCompiledPkgJson, getCurrentPkg, getPackageJson, getPreviousPkg, parsePackageInstallRequests, resetCompiledPkgJson, setCompiledPkgJson } from './pm/packageJson' +import * as quotes from '@cohesible/quotes' +import * as analytics from './services/analytics' +import { TfState } from './deploy/state' +import { bundleExecutable, bundlePkg } from './closures' +import { cleanArtifacts, maybeCreateGcTrigger } from './build-fs/gc' +import { buildBinaryDeps, copyIntegrations, createArchive, createPackageForRelease, installExternalPackages, signWithDefaultEntitlements } from './cli/buildInternal' +import { runCommand, which } from './utils/process' +import { transformNodePrimordials } from './utils/convertNodePrimordials' +import { createCompileView, getPreviousDeploymentData } from './cli/views/compile' +import { createSessionContext, getModuleLoader, getSession, shutdownSessions } from './deploy/session' +import { findArtifactByPrefix, getMetadata } from './build-fs/utils' +import { diffFileInLatestCommit, diffIndices, diffObjects } from './build-fs/stats' +import { renderCmdSuggestion } from './cli/commands' +import * as ui from './cli/ui' +import * as bfs from './artifacts' +import { findAllBareSpecifiers, findProviderImports } from './compiler/entrypoints' +import { makeSea, resolveAssets } from './build/sea' +import { createGraphCompiler, createSerializer } from './static-solver' +import { createInstallView } from './cli/views/install' +import { resolveBuildTarget } from './build/builder' +import { createIndexBackup } from './build-fs/backup' +import { homedir } from 'node:os' +import { createBlock, openBlock } from './build-fs/block' +import { seaAssetPrefix } from './bundler' +import { buildWindowsShim } from './zig/compile' + +export { runTask, getLogger } from './logging' + +// IMPORTANT: must use Terraform v1.5.5 or earlier to avoid BSL + +// TODO: create LZ4 async native module +// TODO: https://github.com/pulumi/pulumi/issues/3388 + +// Apart of refactoring story: +// https://github.com/pulumi/pulumi/issues/3389 + +function removeUndefined>(obj: T | undefined): { [P in keyof T]: NonNullable } | undefined { + if (!obj) { + return obj + } + + return Object.fromEntries(Object.entries(obj).filter(([_, v]) => v !== undefined)) as any +} + +export type CombinedOptions = CompilerOptions & DeployOptions & { + forceRefresh?: boolean + cwd?: string + project?: string + program?: string + process?: string +} + +export function shutdown() { + const promises: Promise[] = [] + + promises.push(shutdownSessions()) + promises.push(shutdownRepos()) + promises.push(analytics.shutdown().catch(e => { + getLogger().warn('Failed to flush events', e) + })) + + return Promise.all(promises) +} + + +// TODO: add permissions model to all system APIs e.g. `fs`, `https`, etc. +// This would be very similar to Deno, but we can do so much more with it + + +export async function syncModule(deploymentId: string, bt = getBuildTargetOrThrow()) { + const projectId = await workspaces.getRemoteProjectId(bt.projectId) + if (projectId) { + getLogger().log('Using remote project id', projectId) + await syncRemote(projectId, bt.programId, deploymentId) + } +} + +export async function publish(target: string, opt?: CompilerOptions & DeployOptions & { newFormat?: boolean; archive?: string; dryRun?: boolean; local?: boolean; globalInstall?: boolean; skipInstall?: boolean }) { + if (opt?.archive) { + const packageDir = getWorkingDir() + const dest = path.resolve(packageDir, opt.archive) + const tmpDest = dest.replace(path.extname(opt.archive), '-tmp') + const bt = getBuildTargetOrThrow() + + try { + await createPackageForRelease(packageDir, tmpDest, { environmentName: bt.environmentName }, true, true) + await createArchive(tmpDest, dest, false) + } finally { + await getFs().deleteFile(tmpDest).catch(throwIfNotFileNotFoundError) + } + + return + } + + if (opt?.local) { + await linkPackage({ dryRun: opt?.dryRun, globalInstall: opt?.globalInstall, skipInstall: opt?.skipInstall, useNewFormat: opt?.newFormat }) + } else { + throw new Error(`Publishing non-local packages is not implemented`) + } +} + +async function findOrphans() { + const previousTemplate = await maybeRestoreTemplate() + if (!previousTemplate) { + throw new Error(`No previous template found`) + } + + const state = await readState() + if (!state) { + throw new Error(`Cannot find orphans with no state`) + } + + const orphans: string[] = [] + const resources = new Map() + for (const [k, v] of Object.entries(previousTemplate.resource)) { + for (const [k2, v2] of Object.entries(v)) { + resources.set(`${k}.${k2}`, v2) + } + } + + for (const r of state.resources) { + const k = `${r.type}.${r.name}` + if (!resources.has(k)) { + orphans.push(k) + } + } + + return { + previousTemplate, + orphans, + } +} + +// IMPORTANT: this implementation is incredibly flawed in that it uses the previous template. It can +// potentially change resources if the orphans depend on current resources. +export async function collectGarbageResources(target: string, opt?: CombinedOptions & { dryRun?: boolean }) { + const { orphans, previousTemplate } = await findOrphans() + if (opt?.dryRun) { + printLine(`Destroying (dry-run):`, orphans.join(', ')) + + return + } + + if (orphans.length === 0) { + getLogger().log(`Nothing to remove`) + + return + } + + printLine(`Destroying:`, orphans.join(', ')) + + const session = await getSession(getTargetDeploymentIdOrThrow()) + await session.setTemplate(previousTemplate) + + const res = await session.destroy({ + ...opt, + targetResources: orphans, + }) + + if (res.state) { + const afs = await bfs.getArtifactFs() + await afs.commit(res.state) + } + + if (res.error) { + throw res.error + } +} + +export async function collectGarbage(target: string, opt?: CombinedOptions & { dryRun?: boolean }) { + await cleanArtifacts(undefined, opt?.dryRun) +} + +async function getMergedGraph(templateFile: TfJson) { + const oldTemplate = await maybeRestoreTemplate() + const oldGraph = oldTemplate ? createSymbolGraphFromTemplate(oldTemplate) : undefined + + return createMergedGraph(createSymbolGraphFromTemplate(templateFile), oldGraph) +} + +async function getDeployView(templateFile: TfJson, isDestroy?: boolean) { + return createDeployView(await getMergedGraph(templateFile), isDestroy) +} + + +// Deploying to an active deployment with a different cloud target is very likely to be an error +// If detected, we'll immediately abort and suggest that the previous deployment should be destroyed +// before deploying to a new cloud provider +async function assertSameCloudTarget(template: TfJson) { + // Template was stripped of metadata + const currentTarget = template['//']?.deployTarget + if (!currentTarget) { + return + } + + const previousTemplate = await maybeRestoreTemplate() + const previousTarget = previousTemplate?.['//']?.deployTarget + if (!previousTarget) { + return + } + + if (currentTarget !== previousTarget) { + throw new RenderableError(`Mis-matched deployment targets`, () => { + printLine(colorize('brightRed', 'Detected mis-matched deployment targets')) + printLine(` previous: ${colorize('green', previousTarget)}`) + printLine(` current: ${colorize('red', currentTarget)}`) + printLine() + + const destroyCmd = renderCmdSuggestion('destroy') + printLine(`If this is intentional, run ${destroyCmd} first and try again.`) + printLine() + }) + } +} + +// The current stance is that anything that might change the "remote" application _must_ be +// an explicit/distinct step. It's not okay to automatically apply changes unless the user +// has opted-in through a very clear and obvious mechanism. +// +// Much of the motivation for this decision is because we don't want to surprise people. +// In theory, if someone was doing everything in a self-contained developer account then +// automatic deployments might actually be expected. But we don't know that. +// +// There's also plenty of situations where automatic deployments would get in the way because +// it's just too slow. Terraform does a decent job of minimizing the # of updates needed +// but it still triggers needless updates quite frequently. +// +// Over time, as the tooling matures, I expect automatic deploys (for developers) to become +// the norm. Right now that just isn't a common thing. + +type DeployOpt2 = CombinedOptions & { + tsOptions?: ts.CompilerOptions; + autoDeploy?: boolean; + sessionCtx?: SessionContext; + dryRun?: boolean; + symbols?: string[] + hideNextSteps?: boolean + rollbackIfFailed?: boolean + useOptimizer?: boolean +} + +async function validateTargets(targets: string[]) { + await Promise.all(targets.map(async t => { + const resolved = path.resolve(getWorkingDir(), t) + if (!(await getFs().fileExists(resolved))) { + throw new Error(`No such file found: ${t}`) + } + })) +} + +export async function deploy(targets: string[], opt?: DeployOpt2) { + await validateTargets(targets) + + const programFsHash = opt?.sessionCtx?.buildTarget.programHash + if (!programFsHash && !opt?.targetResources) { + const doCompile = (beforeSynthCommit?: () => Promise) => compile(targets, { incremental: true, skipSummary: true, hideLogs: true, deployTarget: opt?.deployTarget, beforeSynthCommit }) + + const programState = await getProgramFs().readJson<{ needsSynth?: boolean }>('__buildState__.json').catch(throwIfNotFileNotFoundError) + if (programState?.needsSynth) { + await doCompile(() => getProgramFs().writeJson('[#compile]__buildState__.json', { + ...programState, + needsSynth: false, + })) + } else { + // In any kind of machine-to-machine interaction we shouldn't try to be smart like this + // It's better to fail and say that the program should be compiled explicitly first. + // Auto-compile/synth is a feature for humans, not machines. + // + // TODO: we should check stale compilation for `syn test` too + // TODO: also we should limit the staleness check to the subgraph(s) specified by `targets` + const { stale } = await getStaleDeployableSources(targets.length > 0 ? targets : undefined) ?? {} + if (!stale || stale.size > 0) { + await doCompile() + } else if (opt?.deployTarget) { + const prev = await getPreviousPkg() + if (prev?.synapse?.config?.target !== opt.deployTarget) { + await doCompile() + } + } + } + } + + const deploymentId = getTargetDeploymentIdOrThrow() + const session = await getSession(deploymentId, opt?.sessionCtx, { parallelism: 50 }) + const templateFile = await session.templateService.getTemplate() + await assertSameCloudTarget(templateFile) + + const artifactFs = await bfs.getArtifactFs() + getLogger().debug('Starting deploy operation') + + const view = await getDeployView(templateFile) + + if (opt?.symbols) { + const targets = opt.targetResources ??= [] + const graph = createSymbolGraphFromTemplate(templateFile) + for (const s of opt.symbols) { + if (!!graph.getConfig(s)) { + targets.push(s) + continue + } + + const n = getSymbolNodeFromRef(graph, s) + for (const r of n.resources) { + const id = `${r.type}.${r.name}` + if (!targets.includes(id)) { + targets.push(id) + } + } + } + } + + const resolvedOpt: CombinedOptions = { + parallelism: 50, + disableRefresh: !opt?.forceRefresh, + autoApprove: true, + targetFiles: targets, + ...opt, + } + + try { + const result = await runTask('apply', 'deploy', async () => { + return session.apply(resolvedOpt) + }, 1) + + if (result.state) { + await artifactFs.commit(result.state, programFsHash, opt?.useTests ? true : undefined) + } + + if (result.error && opt?.rollbackIfFailed) { + await shutdownSessions() + await rollback('', opt) + } + + throwIfFailed(view, result.error) + + if (!opt?.hideNextSteps) { + await showNextStepsAfterDeploy(templateFile, result.state) + } + + return result.state + } finally { + if (opt?.syncAfter) { + await syncModule(deploymentId) + } + + view.dispose() + } +} + +function throwIfFailed(view: { formatError: (err: any) => string }, sessionErr: any) { + if (!sessionErr) { + return + } + + if (!(sessionErr instanceof SessionError)) { + throw sessionErr + } + + throw new RenderableError(sessionErr.message, () => { + printLine() + printLine(colorize('red', 'Failed to deploy')) // FIXME: use `destroy` when destroying + + for (const err of sessionErr.errors) { + printLine(view.formatError(err)) + } + }) +} + +async function showNextStepsAfterDeploy(template: TfJson, state: TfState) { + const files = await getEntrypointsFile() + const executables = files?.executables + if (!executables) { + return + } + + const names = Object.keys(executables) + if (names.length !== 1) { + return + } + + const executable = names[0] + const deployed = findDeployedFileResources(state) + if (!deployed[executable]) { + return + } + + printLine() + printLine(`${colorize('brightWhite', executable)} can now be ran with ${renderCmdSuggestion('run')}`) +} + +export async function install(targets: string[], opt?: { dev?: boolean; mode?: 'all' | 'types'; remove?: boolean }) { + const cwd = getBuildTarget()?.workingDirectory ?? process.cwd() + if (cwd !== process.cwd()) { + printLine(colorize(`yellow`, `Treating ${getBuildTarget()?.workingDirectory} as the working directory`)) + } + + const view = createInstallView() + const pkg = await getCurrentPkg() // FIXME: this needs to update the stored package when it changes + if (!pkg) { + if (opt?.remove) { + printLine('Nothing to remove') //FIXME: needs better UX + return + } + if (targets.length === 0) { + printLine('Nothing to install') //FIXME: needs better UX + return + } + + const deps = parsePackageInstallRequests(targets) + await downloadAndUpdatePackage({ directory: cwd, data: { dependencies: deps } as any }, deps) + view.summarize() + + return + } + + if (targets.length === 0) { + if (opt?.remove) { + printLine('Nothing to remove') //FIXME: needs better UX + return + } + + await maybeDownloadPackages(pkg, false, true) + } else { + if (opt?.remove) { + await downloadAndUpdatePackage(pkg, Object.fromEntries(targets.map(k => [k, k])), opt?.dev, opt?.mode === 'all', true) + } else { + const parsed = parsePackageInstallRequests(targets) + await downloadAndUpdatePackage(pkg, parsed, opt?.dev, opt?.mode === 'all') + } + } + + view.summarize() +} + +// deploy/destroy should accept symbol names/ids too **but only if they are unambiguous** +// Using a switch/option to specify symbols works too, it's not as smooth though. +// +// In any case, we should never ever EVER destroy something the user did not want to destroy. +// Seriously. While many resources are easily replaced, some are not. And it's practically +// guaranteed that people will be manually destroying resources in production regardless of +// whatever we say. + +export async function destroy(targets: string[], opt?: CombinedOptions & { dryRun?: boolean; symbols?: string[]; deploymentId?: string }) { + // TODO: this should be done prior to running any commands, not within a command + if (opt?.deploymentId) { + Object.assign(getBuildTargetOrThrow(), { deploymentId: opt?.deploymentId }) + } + + const deploymentId = opt?.deploymentId ?? getTargetDeploymentIdOrThrow() + const state = await readState() + if (!state || state.resources.length === 0) { + getLogger().debug('No resources to destroy, returning early') + printLine(colorize('green', 'Nothing to destroy!')) + return + } + + const template = await maybeRestoreTemplate() + if (!template) { + throw new Error(`No previous deployment template found`) + } + + const programHash = await getPreviousDeploymentProgramHash() + const sessionCtx = await createSessionContext(programHash) + + const session = await getSession(deploymentId, sessionCtx, { parallelism: 50 }) + + const view = await getDeployView(template, true) + const artifactFs = await bfs.getArtifactFs() + + const resolvedOpt: DeployOptions = { + parallelism: 50, + disableRefresh: !opt?.forceRefresh, + targetFiles: targets, + ...opt, + } + + if (opt?.symbols) { + const graph = createSymbolGraphFromTemplate(template) + const sym = getSymbolNodeFromRef(graph, opt.symbols[0]) + const arr = resolvedOpt.targetResources ??= [] + for (const r of sym.resources) { + arr.push(`${r.type}.${r.name}`) + } + } + + try { + const result = await session.destroy(resolvedOpt) + + await artifactFs.commit(result.state, undefined, opt?.useTests ? true : undefined) + + throwIfFailed(view, result.error) + + // Only delete this on a "full" destroy + if (result.state.resources.length === 0) { + const templateFilePath = await session.templateService.getTemplateFilePath() + const stateFile = path.resolve(path.dirname(templateFilePath), '.terraform', 'terraform.tfstate') + await getFs().deleteFile(stateFile).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + }) + + const artifactFs = await bfs.getArtifactFs() + await artifactFs.resetManifest(deploymentId) + } + + return result.state + } finally { + if (opt?.syncAfter) { + await syncModule(deploymentId) + } + + view.dispose() + } +} + +async function getLocallyDeployableResources(session: BoundTerraformSession) { + const templateFile = await session.templateService.getTemplate() + + const configs: Record = {} + for (const [k, v] of Object.entries(templateFile.resource)) { + for (const [k2, v2] of Object.entries(v)) { + configs[`${k}.${k2}`] = v2 + } + } + for (const [k, v] of Object.entries(templateFile.data)) { + for (const [k2, v2] of Object.entries(v)) { + configs[`data.${k}.${k2}`] = v2 + } + } + for (const [k, v] of Object.entries(templateFile.locals)) { + configs[`local.${k}`] = v + } + + const synapseResources = Object.keys(configs) + .filter(x => x.startsWith('synapse_resource.') || x.startsWith('data.synapse_resource.')) + .filter(k => configs[k].type !== 'Example' && configs[k].type !== 'Custom') + + const state = await session.getState() + const deployed = new Set() + if (state) { + for (const r of state.resources) { + deployed.add(`${r.type}.${r.name}`) + } + } + + const deps: Record> = {} + async function loadDeps(targets: string[]) { + const needRefs = targets.filter(x => !deps[x]) + if (needRefs.length === 0) { + return + } + + const allRefs = await session.getRefs(needRefs) + for (const [k, v] of Object.entries(allRefs)) { + deps[k] = new Set(v.map(x => x.subject)) + } + } + + + const canUse = new Map() + const csSet = new Set(synapseResources) + const requiredDeps = new Map() + + // `a` depends on `b` + function addRequiredDep(a: string, b: string) { + if (!requiredDeps.has(a)) { + requiredDeps.set(a, []) + } + requiredDeps.get(a)!.push(b) + } + + function explain(r: string): [string, any[]?] { + return [r, requiredDeps.get(r)?.map(explain)] + } + + // Assumption: no circular deps + async function visit(k: string) { + if (canUse.has(k)) { + return canUse.get(k)! + } + + if (!k.startsWith('data.') && !k.startsWith('local.') && !csSet.has(k)) { + const isDeployed = deployed.has(k) + canUse.set(k, isDeployed) + + return isDeployed + } + + await loadDeps([...deps[k]]) + + for (const d of deps[k]) { + if (!(await visit(d))) { + canUse.set(k, false) + addRequiredDep(k, d) + + return false + } + } + + canUse.set(k, true) + + return true + } + + await loadDeps([...csSet]) + const result: string[] = [] + for (const k of csSet) { + if (await visit(k)) { + result.push(k) + } + } + + return { + result, + deployed, + explain, + } +} + +// FIXME: this is broken when nothing has been deployed +export async function deployModules(targetFiles: string[]) { + const deploymentId = getTargetDeploymentIdOrThrow() + const session = await getSession(deploymentId) + const template = await session.templateService.getTemplate() + const graph = createSymbolGraphFromTemplate(template) + + const set = targetFiles.length > 0 ? new Set(targetFiles) : undefined + const targets: string[] = [] + const { result } = await getLocallyDeployableResources(session) + for (const r of result) { + const type = graph.getResourceType(r) + if (type.kind === 'synapse' && type.name === 'Closure') { + const config = graph.getConfig(r) as any + if (!config?.input?.options?.isModule) continue + + const parsed = parseModuleName(config.module_name) + if (set && set.has(parsed.fileName)) { + targets.push(r) + } + } + } + + if (targets.length === 0) { + throw new Error(`Nothing to deploy`) + } + + getLogger().log('Deploying', targets) + + await deploy([], { + targetResources: targets, + }) +} + +export async function findLocalResources(targets: string[], opt?: CombinedOptions) { + const deploymentId = getTargetDeploymentIdOrThrow() + const session = await getSession(deploymentId) + const template = await session.templateService.getTemplate() + const graph = createSymbolGraphFromTemplate(template) + + const { result, deployed, explain } = await getLocallyDeployableResources(session) + for (const r of result) { + const type = graph.getResourceType(r) + if (type.kind === 'synapse' && type.name === 'Closure') { + const config = graph.getConfig(r) as any + if (!config?.input?.options?.isModule) continue + + printLine('', config.module_name) + } + } + + const canDeployLocally = new Set(result) + for (const x of graph.getSymbols()) { + let isSelfDeployableLocally = true + const n = new Set() + const q = new Set() + for (const r of x.value.resources) { + if (deployed.has(`${r.type}.${r.name}`)) continue + if (canDeployLocally.has(`${r.type}.${r.name}`)) continue + + const z = explain(`${r.type}.${r.name}`) + v(z) + + function v(z: any) { + if (z[1] !== undefined) { // Means the resource depends on a not deployed resource + for (const t of z[1]) { + v(t) + } + } else { + const sym = graph.findSymbolFromResourceKey(z[0]) + if (sym === x){ + isSelfDeployableLocally = false + } else if (sym) { + n.add(sym) + } + q.add(z[0]) + } + } + } + + if (n.size > 0) { + printLine(x.value.name, 'depends on', [...n].map(x => x.value.name).join(', ')) + printLine(` ${[...q].map(x => x.split('--').slice(0, -1).join('--')).map(x => x.slice(x.length - 20)).join(', ')}`) + } else if (!isSelfDeployableLocally) { + printLine(x.value.name) + printLine(` ${[...q].map(x => x.split('--').slice(0, -1).join('--')).map(x => x.slice(x.length - 20)).join(', ')}`) + } + } +} + +// This assumes that the "primary" deployment is already active +export async function runTests(targets: string[], opt?: DeployOptions & { destroyAfter?: boolean; targetIds?: number[]; rollbackIfFailed?: boolean }) { + // TODO: handle the other cases + // await validateTargetsForExecution(targets[0], (await getEntrypointsFile())?.deployables ?? {}) + await compileIfNeeded(targets[0], false) + + const deploymentId = getTargetDeploymentIdOrThrow() + + const filter = { + targetIds: opt?.targetIds, + fileNames: targets.length > 0 ? targets : undefined, + } + + const session = await getSession(deploymentId) + + const suites = await listTestSuites(session.templateService, filter) + const tests = await listTests(session.templateService, filter) + + const targetResources = [ + ...Object.keys(suites), + ...Object.keys(tests), + ] + + const targetModules = new Set() + + // FIXME: figure out a way to avoid doing this. Right now this is done to ensure + // that any resources that cause "side-effects" are also deployed + const suiteIds = [ + ...Object.values(suites).map(x => x.id), + ...Object.values(tests).map(x => x.parentId), + ].filter(isNonNullable) + const resources = (await session.templateService.getTemplate()).resource + for (const [k, v] of Object.entries(resources)) { + for (const [k2, v2] of Object.entries(v as any)) { + const parsed = parseModuleName((v2 as any).module_name) + const key = `${k}.${k2}` + if (parsed.testSuiteId && suiteIds.includes(parsed.testSuiteId) && !targetResources.includes(key)) { + targetResources.push(key) + targetModules.add(parsed.fileName) + } + } + } + + const [currentHash, hashes] = await Promise.all([ + getProgramHash(), + getResourceProgramHashes(getDeploymentFs()) + ]) + + const staleResources = new Set() + for (const r of targetResources) { + if (hashes?.[r] !== currentHash) { + staleResources.add(r) + } + } + + if (targetModules.size === 0) { + // nothing to run?? + throw new RenderableError('Nothing to run', () => { + printLine(colorize('brightRed', 'No test files found')) + }) + } + + // const status = await getDeploymentStatus2([...targetModules], (await getEntrypointsFile())?.deployables ?? {}) + if (staleResources.size === 0) { + getLogger().log('No changes detected, skipping deploy for test resources') + } else { + await deploy([], { + ...opt, + autoApprove: true, + useTests: true, + targetResources: [...staleResources], + hideNextSteps: true, + }) + printLine() // Empty line to separate the deployment info from tests + } + + const testRunner = createTestRunner(session.moduleLoader) + const resolvedSuites = await testRunner.loadTestSuites(suites, tests) + + const failures: FailedTestEvent[] = [] + getLogger().onTest(ev => { + if (ev.status === 'failed') { + failures.push(ev) + } + }) + + const view = createTestView() + + try { + return await testRunner.runTestItems(Object.values(resolvedSuites).filter(x => !x.parentId)) + } finally { + const shouldRollback = opt?.rollbackIfFailed && failures.length > 0 + if (opt?.destroyAfter) { + await destroy([], { + ...opt, + autoApprove: true, + useTests: true, + targetResources: targetResources, + }).catch(err => { + // Rolling back is much more important than a clean destruction of test resources + if (!shouldRollback) { + throw err + } + + getLogger().error(`Failed to destroy test resources`, err) + }) + } + + if (shouldRollback) { + await shutdownSessions() // TODO: can be removed if test resources use a separate process ID + await rollback('', opt) + } + + process.exitCode = failures.length > 0 ? 1 : 0 + + view.showFailures(failures) + view.dispose() + } +} + +export async function testGlob(patterns: string[], opt?: DeployOptions) { + const excluded = undefined + + await runTask('glob', 'glob', async () => { + const res = await glob(getFs(), getWorkingDir(), patterns, excluded ? [excluded] : undefined) + printJson(res) + }, 25) +} + +async function openInEditor(filePath: string) { + const termProgram = process.env['TERM_PROGRAM'] // or $VISUAL or $EDITOR + + switch (termProgram) { + case 'vscode': + return runCommand('code', [filePath]) + + default: + throw new Error(`Not supported: ${termProgram}`) + } +} + +export async function showLogs(patterns: string, opt?: DeployOptions) { + if (patterns === 'list') { + const logs = await getSortedLogs() + for (const l of logs.reverse()) { + printLine(replaceWithTilde(l.filePath)) + } + + return + } + + const latest = await getMostRecentLogFile() + if (!latest) { + return printLine('No log file found') + } + + process.stdout.write(await getFs().readFile(latest)) +} + +export async function plan(targets: string[], opt?: DeployOptions & { symbols?: string[]; forceRefresh?: boolean; planDepth?: number }) { + const session = await getSession(getTargetDeploymentIdOrThrow(), undefined, { ...opt, noSave: true }) + const template = await session.templateService.getTemplate() + + if (opt?.symbols) { + const targets = opt.targetResources ??= [] + const graph = createSymbolGraphFromTemplate(template) + for (const s of opt.symbols) { + if (!!graph.getConfig(s)) { + targets.push(s) + continue + } + + const n = getSymbolNodeFromRef(graph, s) + for (const r of n.resources) { + const id = `${r.type}.${r.name}` + if (!targets.includes(id)) { + targets.push(id) + } + } + } + } + + const res = await session.plan( + { + targetFiles: targets, + consoleLogger: true, + disableRefresh: !opt?.forceRefresh, + ...opt, + // useCachedPlan: true, + } + ) + + const g = await getMergedGraph(template) + const info = extractSymbolInfoFromPlan(g, res) + if (info.size === 0){ + printLine('No changes planned') + return + } + + const groups = groupSymbolInfoByFile(info) + for (const [fileName, group] of Object.entries(groups)) { + // const relPath = path.relative(getWorkingDir(), fileName) + // const headerSize = Math.min(process.stdout.columns, 80) + // const padding = Math.floor((headerSize - (relPath.length + 2)) / 2) + // printLine(colorize('gray', `${'-'.repeat(padding)} ${relPath} ${'-'.repeat(padding)}`)) + // for (const [k, v] of group) { + // printLine(renderSymbolWithState(k.value, v, undefined, ui.spinners.empty)) + // } + printSymbolTable(group) + } +} + +export async function backup(dest: string) { + await createIndexBackup(path.resolve(dest)) +} + +export async function explain(target: string, opt?: DeployOptions & { forceRefresh?: boolean }) { + const session = await getSession(getTargetDeploymentIdOrThrow(), undefined, { ...opt, noSave: true }) + const template = await session.templateService.getTemplate() + const newSourceMap = template['//']?.sourceMap + if (!newSourceMap) { + throw new Error(`No new source map found`) + } + + const newResources: Record = {} + for (const [k, v] of Object.entries(template.resource)) { + for (const [k2, v2] of Object.entries(v)) { + const id = `${k}.${k2}` + newResources[id] = v2 + } + } + + const symbolGraph = createSymbolGraphFromTemplate(template) + const s = getSymbolNodeFromRef(symbolGraph, target) + + const res = await session.plan( + { + consoleLogger: true, + disableRefresh: !opt?.forceRefresh, + ...opt, + targetResources: s.resources.map(x => `${x.type}.${x.name}`), + } + ) + + const edges: [string, string][] = [] + for (const [k, v] of Object.entries(res)) { + if (!v.state?.dependencies) { + continue + } + + for (const d of v.state.dependencies) { + if (d in res) { + edges.push([d, k]) + } + } + } + + const notRoots = new Set(edges.map(e => e[1])) + const roots = new Set(edges.filter(x => !notRoots.has(x[0])).map(x => x[0])) + + for (const r of roots) { + printLine(r, JSON.stringify(getDiff(res[r].change), undefined, 4)) + } +} + +export async function show(targets: string[], opt?: DeployOptions) { + const state = await readState() + if (!state) { + throw new Error('No state to show') + } + + if (targets.length === 0) { + printJson(state) + return + } + + const template = await maybeRestoreTemplate() + if (!template) { + throw new Error(`No deployment template found`) + } + + const graph = createSymbolGraphFromTemplate(template) + const sym = getSymbolNodeFromRef(graph, targets[0]) + const isDebug = (opt as any)?.debug + + for (const r of sym.resources) { + if (r.name.endsWith('--definition') && r.subtype && !isDebug) { + continue + } + + const inst = state.resources.find(r2 => r2.name === r.name && r2.type === r.type) + if (!inst) { + printLine(`${r.name} [missing]`) + continue + } + + const instState = inst.state ?? ((inst as any).instances[0] as typeof inst.state) + const attr = r.subtype ? instState.attributes.output.value : instState.attributes + + const displayName = isDebug ? r.name : (r.subtype ?? r.type) + printLine(displayName) + printJson(attr) + printLine() + } +} + +export async function quote() { + const data = await quotes.getRandomQuote() + + printLine() + printLine(data.text) + printLine() + printLine(colorize('gray', ` —— ${data.author}`)) // 2 em dashes — + printLine() +} + +export async function putSecret(secretType: string, value: string, expiresIn?: number, opt?: CombinedOptions) { + await secrets.putSecret(secretType, value) +} + +export async function getSecret(secretType: string, opt?: CombinedOptions) { + const resp = await secrets.getSecret(secretType) + getLogger().log(resp) +} + +export async function replaceResource(targetModule: string, resourceId: string, opt?: CombinedOptions) { + const session = await getSession(getTargetDeploymentIdOrThrow()) + + const artifactFs = await bfs.getArtifactFs() + + // try { + // const result = await session.apply({ ...options, replaceResource: resourceId, targetResource: resourceId }) + + // await artifactFs.commit(result.state) + + // if (result.error) { + // throw result.error + // } + + // return result.state + // } finally { + // } +} + +function findDeployedFileResources(state: TfState) { + const resources: Record = {} + for (const r of state.resources) { + if (r.type !== 'synapse_resource') continue + + const attr = ('instances' in r) ? (r as any).instances[0].attributes : r.state.attributes + if (attr.type !== 'Closure') continue + + const input = attr.input.value + if (input.options?.isModule) { + resources[input.source] = r + } + } + + return resources +} + +function findDeployedFile(sourcefile: string, state: TfState) { + const resources = findDeployedFileResources(state) + + return resources[sourcefile]?.name +} + +async function resolveReplTarget(target?: string) { + if (!target) { + return + } + + const x = path.relative(getWorkingDir(), target) + const state = await readState() + const rt = state ? findDeployedFile(x, state) : undefined + if (!rt) { + // TODO: it should be possible to use a file that wasn't "deployed" + // We need to check if the target file was a deployable or not + throw new Error(`No file found: ${x}`) + } + + const r = await readResourceState(rt) + + return r.destination +} + + +// TODO: split build artifact template from deployment artifacts. This is mostly to make development faster. +// TODO: detect when mutable references that aren't cloud constructs get split across multiple files + +function parseSymbolRef(ref: string) { + const parsed = ref.match(/^(?:(?[^#]+)#)?(?[^\.\[\]]+)(?:\.(?[^\.\[\]]+))?(?:\[(?[0-9]+)\])?$/) + if (!parsed || !parsed.groups) { + throw new Error(`Failed to parse resource ref: ${ref}`) + } + + const name = parsed.groups['name']! + const fileName = parsed.groups['fileName'] + const attribute = parsed.groups['attribute'] + const index = parsed.groups['index'] ? Number(parsed.groups['index']) : undefined + + return { name, fileName, attribute, index } +} + +function getSymbolNodeFromRef(graph: SymbolGraph, ref: string) { + const { name, fileName, index } = parseSymbolRef(ref) + const matched = graph.matchSymbolNodes(name, fileName) + if (matched.length === 0) { + throw new Error(`No resources found matching name "${name}"${fileName ? ` in file "${fileName}"` : ''}`) + } + + if (matched.length > 1 && index === undefined) { + throw new Error(`Ambiguous match:\n${matched.map(n => ' ' + renderSymbol(n.value, true, true)).join('\n')}`) + } + + return matched[index ?? 0].value +} + +export async function deleteResource(id: string, opt?: CombinedOptions & { dryRun?: boolean; force?: boolean }) { + async function _deleteResource(id: string) { + const state = await readState() + if (!state) { + return + } + + const r = state.resources.findIndex(r => `${r.type}.${r.name}` === id) + if (r === -1) { + getLogger().log(`No resource found`) + return + } + + state.resources.splice(r, 1) + await putState(state) + } + + if (id === 'ALL_CUSTOM') { + const state = await readState() + if (!state) { + return + } + + for (const r of state.resources) { + if (r.type === 'synapse_resource' && (r.name.endsWith('--Example')) || r.name.endsWith('--Custom')) { + await _deleteResource(`${r.type}.${r.name}`) + } + } + + return + } + + if (opt?.force) { + getLogger().log(`Treating target as an absolute reference`) + await _deleteResource(id) + + return + } + + const template = await maybeRestoreTemplate() + if (!template) { + getLogger().warn(`No template found. Treating target as an absolute reference.`) + await _deleteResource(id) + + return + } + + const graph = createSymbolGraphFromTemplate(template) + if (graph.hasResource(id)) { + getLogger().log(`Treating target as an absolute reference`) + await _deleteResource(id) + + return + } + + const s = getSymbolNodeFromRef(graph, id) + + getLogger().log(`Found symbol${opt?.dryRun ? [' [DRY RUN]:'] : ':'}`) + getLogger().log(renderSymbol(s, true, true)) + for (const r of s.resources) { + getLogger().log(' ' + `${r.type}.${r.name}`) + } + + if (!opt?.dryRun) { + getLogger().log('Deleting resource states...') + for (const r of s.resources) { + await _deleteResource(`${r.type}.${r.name}`) + } + } +} + + +export async function taint(id: string, opt?: CombinedOptions & { dryRun?: boolean }) { + async function markTainted(ids: string[]) { + const state = await readState() + if (!state) { + return + } + + for (const id of ids) { + const r = state.resources.find(r => `${r.type}.${r.name}` === id) + if (!r) { + continue + } + + if (r.state) { + r.state.status = 'tainted' + } else { + (r as any).instances[0].status = 'tainted' + } + } + + await putState(state) + } + + const template = await maybeRestoreTemplate() + if (!template) { + getLogger().warn(`No template found.`) + + return + } + + const graph = createSymbolGraphFromTemplate(template) + if (graph.hasResource(id)) { + getLogger().log(`Treating target as an absolute reference`) + await markTainted([id]) + + return + } + + const s = getSymbolNodeFromRef(graph, id) + + getLogger().log(`Found symbol${opt?.dryRun ? [' [DRY RUN]:'] : ':'}`) + getLogger().log(renderSymbol(s, true, true)) + for (const r of s.resources) { + getLogger().log(' ' + `${r.type}.${r.name}`) + } + + if (!opt?.dryRun) { + getLogger().log('Tainting...') + await markTainted(s.resources.map(r => `${r.type}.${r.name}`)) + } +} + + +export async function watch(targets?: string[], opt?: CompilerOptions & { autoDeploy?: boolean }) { + const session = await startWatch(targets, opt) + + await new Promise((resolve, reject) => { + process.on('SIGINT', async () => { + await session.dispose() + resolve() + }) + }) +} + +export async function startWatch(targets?: string[], opt?: CompilerOptions & { autoDeploy?: boolean }) { + const options = { + ...opt, + incremental: true, + } + + const workingDirectory = getWorkingDir() + const resolver = createModuleResolver(getFs(), workingDirectory) + + const sys: ts.System = Object.create(null, Object.getOwnPropertyDescriptors(ts.sys)) + sys.write = s => getLogger().log('[TypeScript]', s.trim()) + sys.writeOutputIsTTY = () => false + sys.clearScreen = () => {} + sys.getCurrentDirectory = () => workingDirectory + + // Needed to find 'lib' files + const isSea = isSelfSea() + const selfPath = getSelfPathOrThrow() + sys.getExecutingFilePath = () => isSea ? selfPath : resolver.resolve('typescript', path.resolve(workingDirectory, 'fake-script.ts')) + + const config = await resolveProgramConfig(options) + config.tsc.cmd.options.noLib = false // Forcibly set this otherwise `watch` can break if `lib` is set + + const watchHost = ts.createWatchCompilerHost( + config.tsc.cmd.fileNames, + config.tsc.cmd.options, + sys, + ts.createEmitAndSemanticDiagnosticsBuilderProgram, + ) + + const sourceMapHost: SourceMapHost & ts.FormatDiagnosticsHost = { + getNewLine: () => sys.newLine, + getCurrentDirectory: watchHost.getCurrentDirectory, + getCanonicalFileName: (ts as any).createGetCanonicalFileName(ts.sys.useCaseSensitiveFileNames) + } + + const tfSession = opt?.autoDeploy + ? await getSession(getTargetDeploymentIdOrThrow()) + : undefined + + const afs = await bfs.getArtifactFs() + async function apply(files?: string[]) { + if (!tfSession) { + return + } + + const result = await tfSession.apply({ + ...options, + parallelism: 50, + autoApprove: true, + targetFiles: files, + disableRefresh: true, + }) + + await afs.commit(result.state) + + if (result.error) { + throw result.error + } + + // XXX + await afs.clearCurrentProgramStore() + } + + let task: Promise | undefined + async function doTask(program: ts.EmitAndSemanticDiagnosticsBuilderProgram, affected: ts.SourceFile[]) { + const builder = createProgramBuilder(config) + const { infraFiles, compiledFiles, compilation } = await builder.emit(program.getProgram(), watchHost, true) + const changedDeployables = new Set() + for (const f of infraFiles) { + const { deps } = getAllDependencies(compilation.graph, [f]) + for (const d of deps) { + if (compiledFiles.has(d)) { + changedDeployables.add(f) + break + } + } + } + getLogger().debug(`Changed infra files:`, [...changedDeployables]) + + if (changedDeployables.size > 0 && config.csc.deployTarget) { + const template = await builder.synth(config.csc.deployTarget) + + await writeTemplate(template) + await commitProgram() + + await tfSession?.setTemplate(template) + + const view = tfSession ? await getDeployView(template) : undefined + + await apply([...changedDeployables].map(f => path.relative(workingDirectory, f))).finally(() => { + view?.dispose() + }) + } else { + await commitProgram() + + // XXX + await afs.clearCurrentProgramStore() + } + } + + const afterProgramCreate = watchHost.afterProgramCreate + + watchHost.afterProgramCreate = program => { + if (task) { + return + } + + const diags = program.getSyntacticDiagnostics() + if (diags.length > 0) { + for (const d of diags) { + const formatted = ts.formatDiagnostic(d, sourceMapHost) + getLogger().error('[TypeScript]', formatted) + } + + return + } + + function collectAffectedFiles() { + const files: ts.SourceFile[] = [] + const diagnostics: ts.Diagnostic[] = [] + + while (true) { + const { affected, result } = program.getSemanticDiagnosticsOfNextAffectedFile() ?? {} + if (!affected) { + break + } + + diagnostics.push(...result!) + + if ((affected as any).kind === ts.SyntaxKind.SourceFile) { + files.push(affected as ts.SourceFile) + } + } + + return { files, diagnostics } + } + + // `affected` will be a superset of all changed files + const affected = collectAffectedFiles() + if (affected.diagnostics.length > 0) { + for (const d of affected.diagnostics) { + const formatted = ts.formatDiagnostic(d, sourceMapHost) + getLogger().error('[TypeScript]', formatted) + } + + return + } + + task = runTask('watch', 'compile', () => doTask(program, affected.files), 100).finally(() => task = undefined) + + return afterProgramCreate?.(program) + } + + const w = ts.createWatchProgram(watchHost) + + return { + dispose: async () => { + await tfSession?.dispose() + w.close() + } + } +} + +async function resolveConfigAndDeps(targets: string[], opt?: CombinedOptions & { skipInstall?: boolean }) { + const config = await resolveProgramConfig(opt, targets.length > 0 ? targets : undefined) + const incrementalHost = createIncrementalHost(config.tsc.cmd.options) + + const deps = await runTask('parse', 'deps', async () => { + return findAllBareSpecifiers(config, await incrementalHost.getTsCompilerHost()) + }, 1) + + await runTask('', 'add implicit deps', async () => { + if (config.pkg) { + const hasSynapseSpecifier = !![...deps].find(spec => spec.startsWith('synapse:')) + if (hasSynapseSpecifier) { + const pkgDeps = (config.pkg as Mutable).devDependencies ??= {} + Object.assign(pkgDeps, await addImplicitPackages(pkgDeps, config.csc as any)) + } + + const hasNodeSpecifier = !![...deps].find(spec => spec.startsWith('node:')) + if (hasNodeSpecifier) { + const pkgDeps = (config.pkg as Mutable).devDependencies ??= {} + if (!pkgDeps['@types/node']) { + // XXX: hard-coded + Object.assign(pkgDeps, { '@types/node': '^20.14.2' }) + } + } + + // We persist the target if there are no other sources + // This is mainly a convenience feature + if (opt?.deployTarget && !config.pkg.synapse?.config?.target) { + const mutable = config.pkg as Mutable + const synapse: Mutable> = mutable.synapse ??= {} + const synapseConfig = synapse.config ??= {} + synapseConfig.target = opt.deployTarget + } + + await setCompiledPkgJson(config.pkg) + } + }, 1) + + if (!opt?.skipInstall && config.pkg) { + const view = createInstallView() + const pkg: ResolvedPackage = { data: config.pkg, directory: getWorkingDir() } + await runTask('package', 'init', () => maybeDownloadPackages(pkg, !!config.csc.noInfra), 1).catch(async e => { + await resetCompiledPkgJson() + throw e + }).finally(() => { + view.dispose() + }) + } else { + getLogger().log('Skipping auto-install') + } + + return { config, deps, incrementalHost } +} + +type CompileOptions = CombinedOptions & { + skipSynth?: boolean + skipInstall?: boolean + skipSummary?: boolean + hideLogs?: boolean + forcedInfra?: string[] + beforeSynthCommit?: () => Promise | void +} + +export async function compile(targets: string[], opt?: CompileOptions) { + const view = createCompileView(opt) + + const { config, incrementalHost } = await resolveConfigAndDeps(targets, opt) + const builder = createProgramBuilder(config, incrementalHost) + const { entrypointsFile } = await runTask('compile', 'all', () => builder.emit(), 100) + + const deployTarget = config.csc.deployTarget + const needsSynth = deployTarget && entrypointsFile.entrypoints.length > 0 + const shouldSkipSynth = config.csc.sharedLib || opt?.skipSynth || config.csc.noSynth || config.csc.noInfra + if (needsSynth && !shouldSkipSynth) { + // Fetch any existing state in the background so we can enhance the output messages + const previousData = !opt?.skipSummary ? getPreviousDeploymentData() : undefined + + const template = await runTask('infra', 'synth', () => builder.synth(deployTarget, entrypointsFile), 10) + const ext = (template as Mutable)['//'] ??= {} + ext.deployTarget = deployTarget // Used to track what target was used in the last deployment + + await writeTemplate(template) + await opt?.beforeSynthCommit?.() + await commitProgram() + + if (!opt?.skipSummary) { + view.showSimplePlanSummary(template, deployTarget, targets, await previousData) + } else { + view.done() + } + } else { + if (needsSynth && opt?.skipSynth) { + await getProgramFs().writeJson('[#compile]__buildState__.json', { + needsSynth: true, + }) + } + await commitProgram() + view.done() + } +} + + +export async function emitBfs(target?: string, opt?: CombinedOptions & { isEmit?: boolean; block?: boolean; outDir?: string; debug?: boolean }) { + if (opt?.isEmit) { + const bt = getBuildTargetOrThrow() + const config = (await getResolvedTsConfig())?.options + const outDir = path.resolve(bt.workingDirectory, opt.outDir ?? config?.outDir ?? 'out') + const normalizedTsOutDir = config?.outDir + ? path.posix.relative(bt.workingDirectory, path.posix.resolve(bt.workingDirectory, config.outDir)) + : undefined + + const dest = await emitPackageDist(outDir, bt, normalizedTsOutDir, config?.declaration) + + const executables = await getExecutables() + if (executables) { + for (const [k, v] of Object.entries(executables)) { + const rel = path.relative(bt.workingDirectory, v) + const outfile = normalizedTsOutDir + ? path.resolve(outDir, path.posix.relative(normalizedTsOutDir, rel)) + : path.resolve(outDir, rel) + + // TODO: this can leave inaccurate source maps if we don't generate source maps here + + await bundleExecutable(bt, v, outfile, bt.workingDirectory, { useOptimizer: (opt as any)?.['no-optimize'] ? false : true }) + } + } + + printLine(colorize('green', `Wrote to ${dest}`)) + + return + } + + if (opt?.block) { + const repo = getDataRepository() + const head = await repo.getHead(getTargetDeploymentIdOrThrow()) + const hash = head?.storeHash + if (!hash) { + throw new Error(`No process found`) + } + + const bfs = await repo.getBuildFs(hash) + const objects = await repo.serializeBuildFs(bfs) + await getFs().writeFile(path.resolve('dist', hash), createBlock(Object.entries(objects))) + + const pHash = await getPreviousDeploymentProgramHash() + if (pHash) { + const bfs = await repo.getBuildFs(pHash) + const objects = await repo.serializeBuildFs(bfs) + await getFs().writeFile(path.resolve('dist', pHash), createBlock(Object.entries(objects))) + } + + return + } + + // XXX: assumes it's a hash + if (target && path.basename(target).length === 64) { + const data = await getFs().readFile(target) + const block = openBlock(Buffer.from(data)) + const index = JSON.parse(block.readObject(path.basename(target)).toString('utf-8')) + + const dest = path.resolve('.vfs-dump') + for (const [k, v] of Object.entries(index.files)) { + await getFs().writeFile(path.resolve(dest, k), block.readObject((v as any).hash)) + } + + return + } + + if (target === 'package') { + await dumpPackage(path.resolve('.vfs-dump')) + } else { + await bfs.dumpFs(target || undefined) + } +} + +export async function inspectBuildTarget(target?: string, opt?: CombinedOptions) { + printJson(getBuildTargetOrThrow()) +} + +export async function dumpArtifacts(target: string, opt?: CombinedOptions) { + const programFs = getProgramFs() + const manifest = await readPointersFile(programFs) + if (!manifest) { + return + } + + + const artifacts = await Promise.all(Object.values(manifest).map(v => Object.values(v)).flat().map(async a => { + const data = Buffer.from(await programFs.readFile(a)).toString('utf-8') + + return { + name: a, + data: JSON.parse(data) as CompiledChunk + } + })) + + const result = artifacts.map(showArtifact).join('\n') + await getFs().writeFile(target, result) +} + +async function emitBlock(id: string, dest: string) { + const repo = getDataRepository() + const head = await repo.getHead(id) + if (!head) { + throw new Error(`No build fs found: ${id}`) + } + + const index = await repo.getBuildFs(head.storeHash) + const data = await repo.serializeBuildFs(index) + const block = createBlock(Object.entries(data)) + await getFs().writeFile(path.resolve(dest, head.storeHash), block) + + return head.storeHash +} + +export async function emitBlocks(dest: string) { + const bt = getBuildTargetOrThrow() + const destDir = path.resolve(getWorkingDir(), dest) + const deploymentId = bt.deploymentId + const ids = { + program: await emitBlock(bt.programId, destDir), + process: deploymentId ? await emitBlock(deploymentId, destDir) : undefined, + } + + await getFs().writeFile( + path.resolve(destDir, 'ids.json'), + JSON.stringify(ids, undefined, 4) + ) +} + +export async function showRemoteArtifact(target: string, opt?: { captured?: boolean; deployed?: boolean; infra?: boolean }) { + const repo = getDataRepository() + + if (target.includes(':')) { + const m = await getMetadata(repo, target) + printJson(m) + + return + } + + const hash = await findArtifactByPrefix(repo, target) + if (!hash) { + //const p = await findParents(target) + //getLogger().log(p) + throw new Error(`No artifact found: ${target}`) + } + + if (!(await repo.hasData(hash))) { + printLine(`Missing data ${hash}`) + + //const p = await findParents(hash) + //getLogger().log(p) + return + } + + const contents = await repo.readData(hash) + const text = Buffer.from(contents).toString('utf-8') + try { + const parsed = JSON.parse(text) + if (opt?.captured) { + const capturedArray = parsed['@@__moveable__']['captured'] + printJson(capturedArray) + + // const t = params.match(/captured:(.*)/)?.[1] + // if (t) { + // printJson(capturedArray[Number(t)]['@@__moveable__']) + // } else { + // printJson(capturedArray) + // } + } else if (opt?.deployed || parsed.kind === 'deployed') { + printLine(Buffer.from(parsed.rendered, 'base64').toString('utf-8')) + + // if (params.includes('imports')) { + // showManifest(parsed.packageDependencies) + // } else { + // printLine(Buffer.from(parsed.rendered, 'base64').toString('utf-8')) + // } + } else if (opt?.infra) { + printLine(Buffer.from(parsed.infra, 'base64').toString('utf-8')) + } else if (parsed.kind === 'compiled-chunk') { + printLine(Buffer.from(parsed.runtime, 'base64').toString('utf-8')) + } else { + printLine(JSON.stringify(parsed, undefined, 4)) + } + } catch { + printLine(text) + } +} + +export async function loadState(target: string, opt?: CombinedOptions) { + const state = JSON.parse(await getFs().readFile(path.resolve(getWorkingDir(), target), 'utf-8')) + await putState(state) +} + +export async function dumpState(target?: string, opt?: CombinedOptions) { + const deploymentId = target ?? getTargetDeploymentIdOrThrow() + const state = await readState(getDeploymentFs(deploymentId)) + await getFs().writeFile( + path.resolve(getWorkingDir(), 'dist', 'states', `${deploymentId}.json`), + JSON.stringify(state, undefined, 4) + ) +} + +// FIXME: exclude invalid move sets (e.g. cycles) +// TODO: add way to add overrides (this command is unlikely to cover every scenario) +// TODO: selectively include files +export async function migrateIdentifiers(targets: string[], opt?: CombinedOptions & { dryRun?: boolean; reset?: boolean }) { + const deploymentId = getBuildTargetOrThrow().deploymentId + const state = deploymentId ? await readState() : undefined + if (!deploymentId || !state || state.resources.length === 0) { + printLine(colorize('brightRed', 'No deployment to migrate')) + return + } + + const targetFiles = targets.length > 0 ? new Set(targets) : undefined + + // TODO: automatically compile if stale + + const session = await getSession(deploymentId) + + const afs = await bfs.getArtifactFs() + const oldTemplate = await afs.maybeRestoreTemplate() + + const oldSourceMap = oldTemplate?.['//']?.sourceMap // FIXME: make this a required field + if (!oldSourceMap) { + throw new Error(`No existing source map found`) + } + + const templateService = session.templateService + const template = await templateService.getTemplate() + const newSourceMap = template['//']?.sourceMap + if (!newSourceMap) { + throw new Error(`No new source map found`) + } + + const newResources: Record = {} + for (const [k, v] of Object.entries(template.resource)) { + for (const [k2, v2] of Object.entries(v)) { + if (targetFiles) { + const parsed = parseModuleName((v2 as any).module_name) + if (!targetFiles.has(parsed.fileName)) continue + } + const id = `${k}.${k2}` + newResources[id] = v2 + } + } + + normalizeConfigs(template) + + // XXX: need to load the state manually + await session.getState() + + const newDeps: Record> = {} + const newRefs = await session.getRefs(Object.keys(newResources)) + for (const [k, v] of Object.entries(newRefs)) { + newDeps[k] = new Set(v.filter(x => !x.subject.startsWith('local.') && !x.subject.startsWith('data.')).map(x => x.subject)) + } + + await session.setTemplate(oldTemplate) + + const oldResources: Record = {} + for (const [k, v] of Object.entries(oldTemplate.resource)) { + for (const [k2, v2] of Object.entries(v)) { + if (targetFiles) { + const parsed = parseModuleName((v2 as any).module_name) + if (!targetFiles.has(parsed.fileName)) continue + } + oldResources[`${k}.${k2}`] = v2 + } + } + + normalizeConfigs(oldTemplate) + + const oldDeps: Record> = {} + const oldRefs = await session.getRefs(Object.keys(oldResources)) + for (const [k, v] of Object.entries(oldRefs)) { + oldDeps[k] = new Set(v.filter(x => !x.subject.startsWith('local.') && !x.subject.startsWith('data.')).map(x => x.subject)) + } + + + if (targetFiles) { + const newKeys = Object.keys(newResources) + const oldKeys = Object.keys(oldResources) + for (const k of Object.keys(newSourceMap.resources)) { + if (!newKeys.includes(k)) { + delete newSourceMap.resources[k] + } + } + for (const k of Object.keys(oldSourceMap.resources)) { + if (!oldKeys.includes(k)) { + delete oldSourceMap.resources[k] + } + } + } + + + // TODO: we _need_ to cross-reference the template with the actual state before proceeding + // TODO: check existing moves + + const moves = runTask('', 'Tree Edits', () => detectRefactors(newResources, newSourceMap, oldResources, oldSourceMap, newDeps, oldDeps), 100) + if (moves.length === 0) { + printLine(colorize('green', 'No resources need to be moved')) + + return + } + + function printMove(move: (typeof moves)[number]) { + const { fromSymbol, toSymbol } = move + const sameFile = fromSymbol.fileName === toSymbol.fileName + + return `${renderSymbol(fromSymbol, !sameFile)} --> ${renderSymbol(toSymbol)}` + } + + function printDedupedMoves(arr: typeof moves) { + const s = new Set() + for (const m of arr) { + s.add(printMove(m)) + } + + return [...s] + } + + if (opt?.dryRun) { + for (const m of printDedupedMoves(moves)) { + printLine(`Will move (dry-run): ${m}`) + } + + return + } + + // XXX: remove the symbol info + const prunedMoves = moves.map(m => ({ from: m.from, to: m.to })) + await saveMoved(prunedMoves) + + for (const m of printDedupedMoves(moves)) { + printLine(`Will move: ${m}`) + } +} + +export async function machineLogin(type?: string, opt?: CombinedOptions) { + const auth = getAuth() + await auth.machineLogin() +} + +export async function getIdentity(type?: string, opt?: CombinedOptions) { + const auth = getAuth() + const acc = await auth.getActiveAccount() + if (!acc) { + throw new Error(`Not logged in`) + } +} + +export async function login(target?: string, opt?: CombinedOptions) { + const auth = getAuth() + const acc = await auth.login(target) +} + +export async function setSessionDuration(target: string, opt?: CombinedOptions) { + const auth = getAuth() + const acc = await auth.getActiveAccount() + if (!acc) { + throw new Error(`Not logged in`) + } + + const sessionDuration = Number(target) + if (isNaN(sessionDuration) || sessionDuration <= 0 || (Math.floor(sessionDuration) !== sessionDuration) || sessionDuration > 7200) { + throw new Error(`Invalid session duration. Must be a non-zero integer between 0 and 7200.`) + } + + await auth.updateAccountConfig(acc, { sessionDuration }) +} + +export async function listProcesses(type?: string, opt?: CombinedOptions) { + const rootDir = workspaces.getRootDir() + const processes = await workspaces.listAllDeployments() + for (const [k, v] of Object.entries(processes)) { + const s = await readState(getDeploymentFs(k, v.programId, v.projectId)) + + const isRunning = s && s.resources.length > 0 + const rel = path.relative(rootDir, v.workingDirectory) + const info = `(${rel || '.'}) [${isRunning ? 'RUNNING' : 'STOPPED'}]` + printLine(`${k} ${info}`) + } +} + +export async function testGcDaemon(type?: string, opt?: CombinedOptions) { + await using trigger = maybeCreateGcTrigger(true) +} + +export async function schemas(type?: string, opt?: CombinedOptions) { + if (type === 'stripe') { + const text = await generateStripeWebhooks() + await getFs().writeFile('webhooks.ts', text) + } else { + const text = await generateOpenApiV3() + await getFs().writeFile('openapiv3.ts', text) + } +} + + +// This inits a new package w/ scaffolding +// `initWorkspace` initializes a non-empty directory +// TODO: we should clone from GitHub +export async function init(opt?: { template?: 'hello-world' | 'react' }) { + const fs = getFs() + const dir = process.cwd() + const dirFiles = (await fs.readDirectory(dir)).filter(f => f.name !== '.git') + if (dirFiles.length !== 0) { + throw new Error(`${dir} is not empty! Move to an empty directory and try again.`) + } + + async function showInstructions(filesCreated: string[]) { + async function detectAwsCredentials() { + if (await getFs().fileExists(path.resolve(homedir(), '.aws'))) { + return true + } + + return ( + process.env['AWS_CONFIG_FILE'] || + process.env['AWS_PROFILE'] || + process.env['AWS_ACCESS_KEY_ID'] || + process.env['AWS_ROLE_ARN'] + ) + } + + const probablyHasAwsCredentials = await detectAwsCredentials() + + printLine(colorize('green', `Created files:`)) + for (const f of filesCreated) { + printLine(colorize('green', ` ${f}`)) + } + if (await getFs().fileExists(path.resolve(dir, 'node_modules'))) { + printLine(colorize('gray', '"node_modules" was created for better editor support')) + } + printLine() + + const deployCmd = renderCmdSuggestion('deploy') + const targetOption = colorize('gray', '--target aws') + + printLine(`You can now use ${deployCmd} to compile & deploy your code!`) + printLine() + printLine(`By default, your code is built for and deployed to a "local" target.`) + + if (probablyHasAwsCredentials) { + printLine(`You can target AWS by adding ${targetOption} to a compile or deploy command.`) + printLine(`The target is remembered for subsequent commands.`) + } else { + const docsLink = colorize('gray', '') + printLine(`Deploying to other targets requires credentials specific to the target.`) + printLine(`For more information, see: ${docsLink}`) + } + } + + if (opt?.template === 'react') { + const tsconfig = { + "include": ["app.tsx"], + "compilerOptions": { + "target": "ES2022", + "jsx": "react-jsx", + "module": "NodeNext", + "moduleResolution": "NodeNext" + }, + } + + await getFs().writeFile(path.resolve(dir, 'tsconfig.json'), JSON.stringify(tsconfig, undefined, 4)) + const pkg = { "synapse": { "dependencies": { "@cohesible/synapse-react": "#synapse-react" } } } + await getFs().writeFile(path.resolve(dir, 'package.json'), JSON.stringify(pkg, undefined, 4)) + const text = 'aW1wb3J0IHsgU3VzcGVuc2UsIHVzZVJlZiB9IGZyb20gJ3JlYWN0JwppbXBvcnQgeyBCdWNrZXQgfSBmcm9tICdzeW5hcHNlOnNybC9zdG9yYWdlJwppbXBvcnQgeyBjcmVhdGVXZWJzaXRlIH0gZnJvbSAnQGNvaGVzaWJsZS9zeW5hcHNlLXJlYWN0JwppbXBvcnQgeyB1c2VTZXJ2ZXIsIG9wZW5Ccm93c2VyIH0gZnJvbSAnQGNvaGVzaWJsZS9zeW5hcHNlLXdlYnNpdGVzJwoKY29uc3Qgd2Vic2l0ZSA9IGNyZWF0ZVdlYnNpdGUoKQpjb25zdCBidWNrZXQgPSBuZXcgQnVja2V0KCkKCmNvbnN0IGdldERhdGEgPSAoa2V5OiBzdHJpbmcpID0+IHsKICAgIHJldHVybiBidWNrZXQuZ2V0KGtleSwgJ3V0Zi04JykuY2F0Y2goZSA9PiB7CiAgICAgICAgcmV0dXJuIChlIGFzIGFueSkubWVzc2FnZQogICAgfSkKfQoKZnVuY3Rpb24gQnVja2V0Q29udGVudHMocHJvcHM6IHsgYnVja2V0S2V5OiBzdHJpbmcgfSkgewogICAgY29uc3QgZGF0YSA9IHVzZVNlcnZlcihnZXREYXRhLCBwcm9wcy5idWNrZXRLZXkpCgogICAgcmV0dXJuIDxwcmU+e2RhdGF9PC9wcmU+Cn0KCmZ1bmN0aW9uIEJ1Y2tldFBhZ2UocHJvcHM6IHsgYnVja2V0S2V5OiBzdHJpbmcgfSkgewogICAgcmV0dXJuICgKICAgICAgICA8ZGl2PgogICAgICAgICAgICA8U3VzcGVuc2UgZmFsbGJhY2s9ezxkaXY+bG9hZGluZzwvZGl2Pn0+CiAgICAgICAgICAgICAgICA8QnVja2V0Q29udGVudHMgYnVja2V0S2V5PXtwcm9wcy5idWNrZXRLZXl9Lz4KICAgICAgICAgICAgPC9TdXNwZW5zZT4KICAgICAgICA8L2Rpdj4KICAgICkKfQoKZnVuY3Rpb24gUm9vdExheW91dCh7IGNoaWxkcmVuIH06IHsgY2hpbGRyZW46IEpTWC5FbGVtZW50IHwgSlNYLkVsZW1lbnRbXSB9KSB7CiAgICByZXR1cm4gKAogICAgICAgIDxodG1sIGxhbmc9ImVuIj4KICAgICAgICAgICAgPGhlYWQ+PC9oZWFkPgogICAgICAgICAgICA8Ym9keT57Y2hpbGRyZW59PC9ib2R5PgogICAgICAgIDwvaHRtbD4KICAgICkKfQoKY29uc3QgYWRkRGF0YSA9IHdlYnNpdGUuYmluZChhc3luYyAoa2V5OiBzdHJpbmcsIGRhdGE6IHN0cmluZykgPT4gewogICAgYXdhaXQgYnVja2V0LnB1dChrZXksIGRhdGEpCn0pCgpmdW5jdGlvbiBCdWNrZXRGb3JtVGhpbmcoKSB7CiAgICBjb25zdCBrZXlSZWYgPSB1c2VSZWY8SFRNTElucHV0RWxlbWVudD4oKQogICAgY29uc3QgdmFsdWVSZWYgPSB1c2VSZWY8SFRNTElucHV0RWxlbWVudD4oKQoKICAgIGZ1bmN0aW9uIHN1Ym1pdCgpIHsKICAgICAgICBjb25zdCBrZXkgPSBrZXlSZWYuY3VycmVudC52YWx1ZQogICAgICAgIGNvbnN0IHZhbHVlID0gdmFsdWVSZWYuY3VycmVudC52YWx1ZQoKICAgICAgICBhZGREYXRhKGtleSwgdmFsdWUpLnRoZW4oKCkgPT4gewogICAgICAgICAgICB3aW5kb3cubG9jYXRpb24gPSB3aW5kb3cubG9jYXRpb24KICAgICAgICB9KQogICAgfQoKICAgIHJldHVybiAoCiAgICAgICAgPGRpdj4KICAgICAgICAgICAgPGxhYmVsPgogICAgICAgICAgICAgICAgS2V5CiAgICAgICAgICAgICAgICA8aW5wdXQgdHlwZT0ndGV4dCcgcmVmPXtrZXlSZWZ9PjwvaW5wdXQ+CiAgICAgICAgICAgIDwvbGFiZWw+CiAgICAgICAgICAgIDxsYWJlbD4KICAgICAgICAgICAgICAgIFZhbHVlCiAgICAgICAgICAgICAgICA8aW5wdXQgdHlwZT0ndGV4dCcgcmVmPXt2YWx1ZVJlZn0+PC9pbnB1dD4KICAgICAgICAgICAgPC9sYWJlbD4KICAgICAgICAgICAgPGJ1dHRvbiBvbkNsaWNrPXtzdWJtaXR9IHN0eWxlPXt7IG1hcmdpbkxlZnQ6ICcxMHB4JyB9fT5BZGQgSXRlbTwvYnV0dG9uPgogICAgICAgIDwvZGl2PgogICAgKQp9Cgphc3luYyBmdW5jdGlvbiBnZXRJdGVtcygpIHsKICAgIHJldHVybiBhd2FpdCBidWNrZXQubGlzdCgpCn0KCmNvbnN0IGRvRGVsZXRlID0gd2Vic2l0ZS5iaW5kKChrZXk6IHN0cmluZykgPT4gYnVja2V0LmRlbGV0ZShrZXkpKQoKZnVuY3Rpb24gQnVja2V0SXRlbShwcm9wczogeyBidWNrZXRLZXk6IHN0cmluZyB9KSB7CiAgICBjb25zdCBrID0gcHJvcHMuYnVja2V0S2V5CgogICAgZnVuY3Rpb24gZGVsZXRlSXRlbSgpIHsKICAgICAgICBkb0RlbGV0ZShrKS50aGVuKCgpID0+IHsKICAgICAgICAgICAgd2luZG93LmxvY2F0aW9uID0gd2luZG93LmxvY2F0aW9uCiAgICAgICAgfSkKICAgIH0KCiAgICByZXR1cm4gKAogICAgICAgIDxsaT4KICAgICAgICAgICAgPGRpdiBzdHlsZT17eyBkaXNwbGF5OiAnZmxleCcsIG1heFdpZHRoOiAnMjUwcHgnLCBtYXJnaW5Cb3R0b206ICcxMHB4JyB9fT4KICAgICAgICAgICAgICAgIDxhIGhyZWY9e2AvYnVja2V0LyR7a31gfSBzdHlsZT17eyBmbGV4OiAnZml0LWNvbnRlbnQnLCBhbGlnblNlbGY6ICdmbGV4LXN0YXJ0JyB9fT57a308L2E+CiAgICAgICAgICAgICAgICA8YnV0dG9uIG9uQ2xpY2s9e2RlbGV0ZUl0ZW19IHN0eWxlPXt7IGFsaWduU2VsZjogJ2ZsZXgtZW5kJyB9fT5EZWxldGU8L2J1dHRvbj4KICAgICAgICAgICAgPC9kaXY+CiAgICAgICAgPC9saT4KICAgICkKfQoKZnVuY3Rpb24gSXRlbUxpc3QoKSB7CiAgICBjb25zdCBpdGVtcyA9IHVzZVNlcnZlcihnZXRJdGVtcykKCiAgICBpZiAoaXRlbXMubGVuZ3RoID09PSAwKSB7CiAgICAgICAgcmV0dXJuIDxkaXY+PGI+VGhlcmUncyBub3RoaW5nIGluIHRoZSBidWNrZXQhPC9iPjwvZGl2PgogICAgfQoKICAgIHJldHVybiAoCiAgICAgICAgPHVsPgogICAgICAgICAgICB7aXRlbXMubWFwKGsgPT4gPEJ1Y2tldEl0ZW0ga2V5PXtrfSBidWNrZXRLZXk9e2t9Lz4pfQogICAgICAgIDwvdWw+CiAgICApCn0KCmZ1bmN0aW9uIEhvbWVQYWdlKCkgewogICAgcmV0dXJuICgKICAgICAgICA8ZGl2PgogICAgICAgICAgICA8QnVja2V0Rm9ybVRoaW5nPjwvQnVja2V0Rm9ybVRoaW5nPgogICAgICAgICAgICA8YnI+PC9icj4KICAgICAgICAgICAgPFN1c3BlbnNlIGZhbGxiYWNrPSdsb2FkaW5nJz4KICAgICAgICAgICAgICAgIDxJdGVtTGlzdC8+CiAgICAgICAgICAgIDwvU3VzcGVuc2U+CiAgICAgICAgPC9kaXY+CiAgICApCn0KCndlYnNpdGUuYWRkUGFnZSgnLycsIHsKICAgIGNvbXBvbmVudDogSG9tZVBhZ2UsCiAgICBsYXlvdXQ6IHsgY29tcG9uZW50OiBSb290TGF5b3V0IH0sCn0pCiAgICAKCndlYnNpdGUuYWRkUGFnZSgnL2J1Y2tldC97YnVja2V0S2V5fScsIHsKICAgIGNvbXBvbmVudDogQnVja2V0UGFnZSwKICAgIGxheW91dDogeyBjb21wb25lbnQ6IFJvb3RMYXlvdXQgfSwKfSkKCmV4cG9ydCBhc3luYyBmdW5jdGlvbiBtYWluKCkgewogICAgb3BlbkJyb3dzZXIod2Vic2l0ZS51cmwpCn0KCg==' + await getFs().writeFile(path.resolve(dir, 'app.tsx'), Buffer.from(text, 'base64')) + await showInstructions(['app.tsx', 'package.json', 'tsconfig.json']) + + return + } + + const text = ` +import { Function } from 'synapse:srl/compute' + +const hello = new Function(() => { + return { message: 'hello, world!' } +}) + +export async function main(...args: string[]) { + console.log(await hello()) +} +`.trimStart() + + await fs.writeFile(path.resolve(dir, 'hello.ts'), text, { flag: 'wx' }) + + await showInstructions(['hello.ts']) +} + +export async function clearCache(targetKey?: string, opt?: CombinedOptions) { + if (!targetKey) { + return clearIncrementalCache() + } + + const programFs = getProgramFs() + await programFs.clear(targetKey) +} + +export async function listCommitsCmd(mod: string, opt?: CombinedOptions & { useProgram?: boolean }) { + const timestampWidth = new Date().toISOString().length + const hashWidth = 12 + const commits = await listCommits(opt?.useProgram ? getBuildTargetOrThrow().programId : undefined) + if (commits.length === 0) { + printLine(colorize('brightRed', 'No commits found')) + return + } + + if (!opt?.useProgram) { + printLine(`${'Timestamp'.padEnd(timestampWidth, ' ')} ${'Process'.padEnd(hashWidth, ' ')} ${'Program'.padEnd(hashWidth, ' ')} ${'IsTest?'}`) + for (const c of commits) { + printLine(c.timestamp, c.storeHash.slice(0, hashWidth), c.programHash?.slice(0, hashWidth), !!c.isTest) + } + } else { + printLine(`${'Timestamp'.padEnd(timestampWidth, ' ')} ${'Program'.padEnd(hashWidth, ' ')}`) + for (const c of commits) { + printLine(c.timestamp, c.storeHash.slice(0, hashWidth)) + } + } +} + +export async function rollback(mod: string, opt?: CombinedOptions) { + printLine(colorize('yellow', 'Rolling back...')) + + const syncAfter = opt?.syncAfter ?? !!getCiType() // XXX: for internal use only + + const commits = await listCommits() + const targetCommit = commits.filter(x => !x.isTest)[1] + if (!targetCommit) { + throw new Error('Nothing to rollback to') + } + + if (!targetCommit.programHash) { + throw new Error(`No program to restore from`) + } + + printLine(`Using previous program hash: ${targetCommit.programHash}`) + const sessionCtx = await createSessionContext(targetCommit.programHash) + + await deploy([], { ...opt, syncAfter, sessionCtx, hideNextSteps: true, rollbackIfFailed: false }) +} + +export async function printTypes(target?: string, opt?: CombinedOptions) { + const config = await resolveProgramConfig(opt) + const builder = createProgramBuilder(config) + await builder.printTypes(target) +} + +export async function processProf(t?: string, opt?: CombinedOptions) { + const buildTarget = getBuildTargetOrThrow() + const afs = await bfs.getArtifactFs() + const programStore = await afs.getCurrentProgramStore().getRoot() + const fs = toFs(buildTarget.workingDirectory, programStore.root, getFs()) + + const target = t ?? await (async function () { + const files = await glob(getFs(), getWorkingDir(), ['*.cpuprofile']) + if (files.length === 0) { + throw new Error(`No ".cpuprofile" files found in current directory`) + } + if (files.length > 1) { + throw new Error(`Ambiguous match: ${files.join(', ')}`) + } + return files[0] + })() + + const r = await loadCpuProfile(fs, target, buildTarget.workingDirectory, await getProjectOverridesMapping(getFs())) + getLogger().log(r) +} + +async function runProgramExecutable(fileName: string, args: string[]) { + const moduleLoader = await runTask('init', 'loader', () => getModuleLoader(), 1) // 8ms on simple hello world no infra + const m = await moduleLoader.loadModule(fileName) + if (typeof m.main !== 'function') { + throw new Error(`Missing main function in file "${fileName}", found exports: ${Object.keys(m)}`) + } + + try { + const exitCode = await m.main(...args) + if (typeof exitCode === 'number') { + process.exitCode = exitCode + } + } catch (e) { + process.exitCode = 1 + printLine(ui.format(e)) + } +} + +async function compileIfNeeded(target?: string, skipSynth = true) { + // XXX: we should normalize everything at program entrypoints + if (isWindows() && target) { + target = target.replaceAll('/', '\\') + } + + // TODO: we can skip synth if the stale files aren't apart of the synthesis dependency graph + // TODO: if `run` automatically compiles anything, it should _always_ use the last-used settings + const { stale, sources } = await getStaleSources() ?? {} + if (!sources || (stale && stale.size > 0) || (target && !sources[target])) { + // We don't need to generate a template, we just want updated program analyses + // TODO: mark the current compilation as "needs synth" + await compile( + target ? [target] : [], + stale ? { incremental: true, skipSynth, skipSummary: true } : { skipSummary: true } + ) + } +} + +async function getDeploymentStatus(target: string, deployables: Record) { + const incr = createIncrementalHost({}) + + const [deps, info, sources] = await Promise.all([ + incr.getCachedDependencies(path.resolve(getWorkingDir(), target)), + getPreviousDeployInfo(), + readSources(), + ]) + + const allDeps = getAllDependencies(deps, [path.resolve(getWorkingDir(), target)]) + const resolvedSource = path.resolve(getWorkingDir(), target) + const deployableSet = new Set(Object.keys(deployables).map(k => path.resolve(getWorkingDir(), k))) + + // BUG: synthesis removes unused imports, but we don't check that here + const isTargetDeployable = deployableSet.has(resolvedSource) + const toCheck = isTargetDeployable ? [resolvedSource] : allDeps.deps + const needsDeploy: string[] = [] + const staleDeploys: string[] = [] + for (const d of toCheck) { + if (!deployableSet.has(d)) continue + + const relPath = path.relative(getWorkingDir(), d) + const resourceName = info?.state ? findDeployedFile(relPath, info.state) : undefined + + if (!resourceName) { + needsDeploy.push(relPath) + continue + } + + // TODO: improve staleness check by only looking at captured symbols + const currentHash = sources?.[relPath].hash + if (currentHash && currentHash !== info?.deploySources?.[relPath].hash) { + staleDeploys.push(relPath) + } + } + + return { + isTargetDeployable, + needsDeploy, + staleDeploys, + sources, + } +} + +async function validateTargetsForExecution(targets: string, deployables: Record) { + const status = await getDeploymentStatus(targets, deployables) + + if (status.needsDeploy.length > 0) { + // TODO: automatically deploy for "local" (or if the user opts-in for other targets) + throw new RenderableError(`Program not deployed`, () => { + function printSuggestion() { + printLine() + + const deployCmd = renderCmdSuggestion('deploy', status.needsDeploy) + printLine(`Run ${deployCmd} first and try again.`) + printLine() + } + + if (status.needsDeploy.length === 1 && status.needsDeploy[0] === targets) { + printLine(colorize('brightRed', 'Resources in the target file need to be deployed')) + printSuggestion() + return + } + + // Implies length >= 2 + if (status.needsDeploy.includes(targets)) { + printLine(colorize('brightRed', 'The target file and its dependencies have not been deployed')) + } else { + if (status.needsDeploy.length === 1) { + printLine(colorize('brightRed', 'Dependency has not been deployed')) + printSuggestion() + return + } + + printLine(colorize('brightRed', 'Dependencies have not been deployed')) + } + + printLine('Needs deployment:') + for (const f of status.needsDeploy) { + printLine(` ${f}`) + } + + printSuggestion() + }) + } + + // If the target file isn't a deployable AND its deps are stale, we will fail + if (status.staleDeploys.length > 0) { + // This is too strict and somewhat incorrect. + // We should fail when a non-deployable file depends on a stale deployable in general. + if (!status.isTargetDeployable) { + throw new RenderableError(`Program not deployed`, () => { + printLine(colorize('brightRed', 'The target\'s dependencies have not been deployed')) + }) + } + printLine(colorize('brightYellow', 'Deployment has not been updated with the latest changes')) + } + + return status +} + +export async function run(name: string | undefined, args: string[], opt?: CombinedOptions & { skipValidation?: boolean; skipCompile?: boolean }) { + const maybeCmd = name ? await maybeGetPkgScript(name) : undefined + if (maybeCmd) { + getLogger().log(`Running package script: ${name}`) + + const runCommand = createNpmLikeCommandRunner(maybeCmd.pkg.directory, undefined, 'inherit') + await runCommand(maybeCmd.cmd, args) + + return + } + + if (!opt?.skipCompile) { + await compileIfNeeded() + } + + const programFs = opt?.skipCompile + ? await getPreviousDeploymentProgramFs() + : undefined + + // Must be loaded after compilation + const files = await getEntrypointsFile(programFs) + + const executables = files?.executables + if (!executables || Object.keys(executables).length === 0) { + throw new Error(`No executables found`) + } + + const deployables = files.deployables ?? {} + + const [source, output] = !name ? Object.entries(executables)[0] : Object.entries(executables).find(([k, v]) => k === name)! + + if (!opt?.skipValidation) { + const status = await validateTargetsForExecution(source, deployables) + + const resolved = status.isTargetDeployable + ? await resolveReplTarget(source) + : path.resolve(getWorkingDir(), output) + + return runProgramExecutable(resolved, args) + } + + return runProgramExecutable(path.resolve(getWorkingDir(), output), args) +} + +// need to add this to a few places to make sure we handle abs paths +function normalizeToRelative(fileName: string, workingDir = getWorkingDir()) { + return path.relative(workingDir, path.resolve(workingDir, fileName)) +} + +export async function replCommand(target: string, opt?: { entrypoint?: string; cwd?: string }) { + await compileIfNeeded(target) + + const repl = await runTask('', 'repl', async () => { + const files = await getEntrypointsFile() + const deployables = files?.deployables ?? {} + const status = await validateTargetsForExecution(target, deployables) + const outfile = status.sources?.[target]?.outfile + if (!status.isTargetDeployable && !outfile) { + throw new RenderableError('No such file', () => { + printLine(colorize('brightRed', 'No such file exists')) + }) + } + + const resolved = status.isTargetDeployable + ? await resolveReplTarget(target) + : path.resolve(getWorkingDir(), outfile!) + + const moduleLoader = await runTask('init', 'loader', () => getModuleLoader(false), 1) // 8ms on simple hello world no infra + + return enterRepl(resolved, moduleLoader, {}) + }, 1) + + return repl.promise +} + +async function getPreviousDeploymentProgramFs() { + const hash = await getPreviousDeploymentProgramHash() + if (!hash) { + return + } + + return toFsFromHash(hash) +} + +async function getPreviousDeployInfo() { + if (!getBuildTargetOrThrow().deploymentId) { + return + } + + const hash = await getPreviousDeploymentProgramHash() + if (!hash) { + return + } + + const state = await readState() + const oldProgramFs = await getFsFromHash(hash) + const deploySources = await readSources(oldProgramFs) + + return { state, hash, deploySources } +} + +async function getStaleSources(include?: Set) { + const sources = await readSources() + const hasher = getFileHasher() + if (!sources) { + return + } + + // Check hashes + const stale = new Set() + const workingDir = getWorkingDir() + + async function checkSource(k: string, v: { hash: string }) { + const source = path.resolve(workingDir, k) + if (include && !include.has(source)) return + + const hash = await hasher.getHash(source).catch(e => { + throwIfNotFileNotFoundError(e) + }) + + if (!hash) { + delete sources![source] + return + } + + if (v.hash !== await hasher.getHash(source)) { + stale.add(source) + } + } + + await Promise.all(Object.entries(sources).map(([k, v]) => checkSource(k, v))) + + return { stale, sources } +} + +// This is used to see if we need to re-synth +async function getStaleDeployableSources(targets?: string[]) { + const deployables = await getDeployables() + if (!deployables) { + return + } + + const deployableSet = new Set(Object.keys(deployables).map(k => path.resolve(getWorkingDir(), k))) + const incr = createIncrementalHost({}) + const deps = await incr.getCachedDependencies(...(deployableSet)) + const allDeps = getAllDependencies(deps, targets?.map(x => path.resolve(getWorkingDir(), x)) ?? [...deployableSet]) + + return getStaleSources(allDeps.deps) +} + +export async function showStatus(opt?: { verbose?: boolean }) { + // Packages (installation) + // Compile (pending moves) + // Deploy + // Projects? + // + + const programFs = getProgramFs() + const installation = await getInstallation(programFs) + if (installation?.packages) { + printLine(colorize('green', 'Installed packages')) + if (opt?.verbose) { + for (const [k, v] of Object.entries(installation.packages)) { + printLine(` ${k} -> ${v.name}${v.version ? `@${v.version}` : ''}`) + } + } + } else { + printLine(colorize('red', 'No packages installed')) + } + + const { stale: staleSources, sources } = await getStaleSources() ?? {} + + if (!staleSources) { + printLine(colorize('red', 'Not compiled')) + } else { + if (staleSources.size > 0) { + printLine(colorize('yellow', 'Stale compilation')) + for (const f of staleSources) { + printLine(colorize('yellow', ` ${path.relative(getWorkingDir(), f)}`)) + } + } else { + printLine(colorize('green', 'Compiled')) + } + } + + const info = await getPreviousDeployInfo() + if (!info?.deploySources || !info.state || info.state.resources.length === 0) { + printLine(colorize('red', 'Not deployed')) + return + } + + const deployables = await getDeployables() ?? {} + const deployableSet = new Set(Object.keys(deployables).map(k => path.resolve(getWorkingDir(), k))) + const incr = createIncrementalHost({}) + const deps = await incr.getCachedDependencies(...deployableSet) + const allDeps = getAllDependencies(deps, [...deployableSet]) + + // This staleness check is probably too simplistic + const stale = new Set() + for (const d of allDeps.deps) { + const source = path.relative(getWorkingDir(), d) + const h = info.deploySources[source]?.hash + if (h && h !== sources![source]?.hash) { + stale.add(source) + } + } + + if (stale.size > 0) { + printLine(colorize('yellow', 'Stale deployment')) + for (const f of stale) { + printLine(colorize('yellow', ` ${path.relative(getWorkingDir(), f)}`)) + } + } else { + printLine(colorize('green', 'Deployed')) + } + + const currentHash = await getProgramHash() + if (currentHash !== info.hash) { + const moved = await getMoved() + if (moved) { + // TODO: show the moves... + printLine(colorize('blue', 'Pending moves')) + } + } +} + +async function maybeGetPkgScript(name: string, fs = getFs()) { + const workingDir = getWorkingDir() + const pkg = await getPackageJson(fs, workingDir, false) + if (!pkg) { + return + } + + const cmd = pkg.data.scripts?.[name] + + return cmd ? { pkg, cmd } : undefined +} + +export async function convertPrimordials(targets: string[]) { + await transformNodePrimordials(targets) +} + +export async function testZip(dir: string, dest?: string) { + dest ??= path.resolve(path.dirname(dir), `${path.basename(dir)}.zip`) + await createZipFromDir(dir, dest, true) +} + +export async function printBlock(hash: string, opt?: any) { + await printBlockInfo(hash) +} + +export async function diffIndicesCmd(a: string, b: string, opt?: any) { + await diffIndices(getDataRepository(), a, b) +} + +export async function diffObjectsCmd(a: string, b: string, opt?: any) { + await diffObjects(getDataRepository(), a, b) +} + +export async function diffFileCmd(fileName: string, opt?: { commitsBack?: number }) { + await diffFileInLatestCommit(fileName, opt) +} + +// This is safe because deployments always reference fs hashes not head IDs +export async function clean(opt?: any) { + await getDataRepository().deleteHead(getBuildTargetOrThrow().programId) +} + +export async function lockedInstall() { + await verifyInstall() +} + +export async function listInstallTree() { + await listInstall() +} + +export async function loadBlock(target: string, dest?: string, opt?: any) { + const data = await getFs().readFile(target) + const block = openBlock(Buffer.from(data)) + const index = JSON.parse(block.readObject(path.basename(target)).toString('utf-8')) + checkBlock(Buffer.from(data), index) + + if (dest) { + printLine(`Loading block to ${dest}`) + } + + const repo = getDataRepository(undefined, dest) + await Promise.all(block.listObjects().map(h => repo.writeData(h, block.readObject(h)))) +} + +export async function inspectBlock(target: string, opt?: any) { + const data = await getFs().readFile(target) + const block = openBlock(Buffer.from(data)) + const objects = block.listObjects().map(h => [h, block.readObject(h).byteLength] as const) + .sort((a, b) => b[1] - a[1]) + + for (const [h, size] of objects) { + printLine(`${h} ${size}`) + } + + const index = JSON.parse(block.readObject(path.basename(target)).toString('utf-8')) + checkBlock(Buffer.from(data), index) + printLine(colorize('green', 'No issues found')) + // printJson(index) + + // const f = index.files['full-state.json'] + // if (f) { + // const h = f.hash + // const sh = f.storeHash ?? index.stores[f.store].hash + // const state = JSON.parse(block.readObject(h).toString('utf-8')) + // printJson(state) + + // const m = JSON.parse(block.readObject(sh).toString('utf-8')) + // printJson(m) + // } +} + +export function runUserScript(target: string) { + const loader = createMinimalLoader(target.endsWith('.ts')) + + return loader.loadModule(target) +} + +export async function buildExecutables(opt: any = {}) { + const bt = getBuildTargetOrThrow() + const pkg = await getCurrentPkg() + if (!pkg) { + throw new Error(`No package.json found`) + } + + const bin = getPkgExecutables(pkg.data) + const executables = await getExecutables() + if (!bin || !executables) { + throw new Error(`No executables to build`) + } + + async function _getNodePath() { + if (isSelfSea()) { + return process.execPath + } + + try { + return await which('synapse') + } catch (e) { + getLogger().log('Failed to find "synapse", falling back to node', e) + + return which('node') + } + } + + const getNodePath = memoize(_getNodePath) + + const set = new Set(Object.values(executables).map(k => path.resolve(bt.workingDirectory, k))) + + const external = ['esbuild', 'typescript', 'postject'] + // XXX: this is hard-coded to `synapse` + const bundleOpt = pkg.data.name === 'synapse' ? { + external, + lazyLoad: ['typescript', 'esbuild', ...lazyNodeModules], + lazyLoad2: ['@cohesible/*'], + } : undefined + + if (pkg.data.name === 'synapse') { + process.env.SKIP_SEA_MAIN = '1' + process.env.CURRENT_PACKAGE_DIR = pkg.directory + } + + const assets: Record = {} + for (const [k, v] of Object.entries(bin)) { + const resolved = path.resolve(bt.workingDirectory, v) + if (!set.has(resolved)) continue + + const res = await bundleExecutable(bt, resolved, undefined, undefined, { sea: true, ...bundleOpt }) + Object.assign(assets, res.assets) + + const dest = path.resolve(bt.workingDirectory, 'dist', 'bin', k) + await makeSea(res.outfile, await getNodePath(), dest, res.assets) + } +} + +export async function convertBundleToSea(dir: string) { + const nodePath = path.resolve(dir, 'bin', process.platform === 'win32' ? 'node.exe' : 'node') + await makeExecutable(nodePath) + + const bundledCliPath = path.resolve(dir, 'dist', 'cli.js') + const assets: Record = {} + const assetsDir = path.resolve(dir, 'assets') + const hasAssets = await getFs().fileExists(assetsDir) + if (hasAssets) { + for (const f of await getFs().readDirectory(assetsDir)) { + if (f.type === 'file') { + assets[`${seaAssetPrefix}${f.name}`] = path.resolve(assetsDir, f.name) + } + } + } + + const seaDest = path.resolve(dir, 'bin', process.platform === 'win32' ? 'synapse.exe' : 'synapse') + await makeSea(bundledCliPath, nodePath, seaDest, assets, true) + + await getFs().deleteFile(nodePath) + await getFs().deleteFile(path.resolve(dir, 'dist', 'cli.js')) + await getFs().deleteFile(path.resolve(dir, 'node_modules')).catch(throwIfNotFileNotFoundError) + if (hasAssets) { + await getFs().deleteFile(path.resolve(dir, 'assets')) + } + + await createArchive(dir, `${dir}${process.platform === 'linux' ? `.tgz` : '.zip'}`, false) +} + +const lazyNodeModules = [ + 'node:vm', + 'node:http', + 'node:https', + 'node:repl', + 'node:assert', + 'node:stream', + 'node:zlib', + 'node:module', + 'node:net', + 'node:url', +] + +export async function internalBundle(target?: string, opt: any = {}) { + const targetDouble = target?.split('-') + const resolved = resolveBuildTarget({ os: targetDouble?.[0] as any, arch: targetDouble?.[1] as any }) + const os = resolved.os + const arch = resolved.arch + + const libc = opt.libc + const external = ['esbuild', 'typescript', 'postject'] + const isProdBuild = opt.production + const dirName = opt.stagingDir ?? `synapse-${os}-${arch}` + const outdir = path.resolve(getWorkingDir(), 'dist', dirName) + + const isSea = opt.sea || opt.seaOnly + const hostTarget = resolveBuildTarget() + if (isSea && (hostTarget.os !== os || hostTarget.arch !== arch)) { + throw new Error(`Cross-platform builds are not supported for snapshots/SEAs`) + } + + const shouldSign = false // opt.sign || isProdBuild + const bundledCliPath = path.resolve(outdir, 'dist/cli.js') + + opt.lto ??= isProdBuild + + const execPath = (name: string) => path.resolve(outdir, 'bin', `${name}${os === 'windows' ? '.exe' : ''}`) + const toolPath = (name: string) => path.resolve(outdir, 'tools', `${name}${os === 'windows' ? '.exe' : ''}`) + + if (opt.integrationsOnly) { + return copyIntegrations(getRootDirectory(), outdir, opt.integration) + } + + const nodePath = execPath('node') + const seaDest = execPath('synapse') + + if (opt.seaOnly) { + const bundle = await createBundle() + + return await makeSea(bundledCliPath, nodePath, seaDest, bundle.assets) + } + + async function bundleMain() { + const bundleOpt = { + external, + minifyKeepWhitespace: isProdBuild, + lazyLoad: (isSea || opt.seaPrep) ? ['typescript', 'esbuild', ...lazyNodeModules] : [], + lazyLoad2: (isSea || opt.seaPrep) ? ['@cohesible/*'] : [], + } + + if (isProdBuild && process.env.SKIP_SEA_MAIN) { + const bt = getBuildTargetOrThrow() + return bundleExecutable( + bt, + 'dist/src/cli/index.js', + bundledCliPath, + bt.workingDirectory, + { sea: true, ...bundleOpt } + ) + } + + return bundlePkg( + 'dist/src/cli/index.js', + getWorkingDir(), // TODO: we should bundle in an empty directory to prevent extraneous files from being used + bundledCliPath, + bundleOpt, + ) + } + + // TODO: external sourcemaps for the bundle/binary + async function createBundle() { + const res = await bundleMain() + + // XXX: Patch the bundle + await getFs().writeFile( + bundledCliPath, + (await getFs().readFile(bundledCliPath, 'utf-8')) + .replace('#!/usr/bin/env node', 'var exports = {};') + ) + + const procFs = getDeploymentFs() + + await getFs().writeFile( + path.resolve(outdir, 'dist', 'completions.sh'), + await procFs.readFile(path.resolve(getWorkingDir(), 'src', 'cli', 'completions', 'completion.sh')) + ) + + return res + } + + async function bundleInstallScript() { + await bundleExecutable( + getBuildTargetOrThrow(), + 'dist/src/cli/install.js', + path.resolve(outdir, 'dist', 'install.js') + ) + } + + await bundleInstallScript() + + await createPackageForRelease(getWorkingDir(), outdir, { + external, + os, + arch, + libc, + lto: opt.lto, + sign: shouldSign, + snapshot: isSea || opt.seaPrep, + stripInternal: isProdBuild, + buildLicense: isProdBuild, + downloadOnly: opt.downloadOnly, + }) + + const bundle = await createBundle() + await copyIntegrations(getRootDirectory(), outdir, opt.integration) + + if (resolved.os === 'windows') { + const shimPath = await buildWindowsShim() + await getFs().writeFile( + path.resolve(outdir, 'dist', 'shim.exe'), + await getFs().readFile(shimPath) + ) + } + + await makeExecutable(nodePath) + await makeExecutable(toolPath('terraform')) + if (isSea) { + await makeExecutable(toolPath('esbuild')) + } + + if (isSea) { + await makeSea(bundledCliPath, nodePath, seaDest, bundle.assets, !shouldSign) + if (isProdBuild || opt.stagingDir || !!getCiType()) { + if (!opt.preserveSource) { + await getFs().deleteFile(bundledCliPath) + } + await getFs().deleteFile(nodePath) + await getFs().deleteFile(path.resolve(outdir, 'node_modules')) + } + + if (shouldSign) { + await signWithDefaultEntitlements(seaDest) + } + } else { + await getFs().deleteFile(path.resolve(outdir, 'node_modules', '.bin')).catch(throwIfNotFileNotFoundError) + } + + if (opt.seaPrep) { + const resolved = await resolveAssets(bundle.assets) + if (resolved) { + const assetsDest = path.resolve(outdir, 'assets') + for (const [k, v] of Object.entries(resolved)) { + await getFs().writeFile( + path.resolve(assetsDest, k), + await getFs().readFile(v) + ) + } + } + } + + // darwin we use `.zip` for signing + const extname = opt.seaPrep || os === 'linux' ? '.tgz' : '.zip' + await createArchive(outdir, `${outdir}${extname}`, shouldSign && !opt.seaPrep) +} + + +// This is to get the WS URL for Inspector +// http://host:port/json/list + + diff --git a/src/loader.ts b/src/loader.ts new file mode 100644 index 0000000..63a791d --- /dev/null +++ b/src/loader.ts @@ -0,0 +1,1803 @@ +import * as path from 'node:path' +import * as vm from 'node:vm' +import * as util from 'node:util' +import { createRequire, isBuiltin } from 'node:module' +import type * as terraform from './runtime/modules/terraform' +import { AsyncLocalStorage } from 'node:async_hooks' +import { runTask, getLogger } from './logging' +import { PackageResolver, PackageService } from './pm/packages' +import { BuildTarget, LocalWorkspace, getV8CacheDirectory } from './workspaces' + + +// BUG: ReferenceError: runtime is not defined +// `scopes.ts` cannot handle multiple imports from the same module (?) +import type { ArtifactFs, Scope } from './runtime/modules/core' +import { ReplacementSymbol, TargetsFile } from './compiler/host' +import { SyncFs } from './system' +import { Module, ModuleCreateOptions, SourceMapParser, createGetCredentials, createModuleLinker, createScriptModule, createSyntheticCjsModule, getArtifactOriginalLocation, registerSourceMapParser } from './runtime/loader' +import { Artifact, BuildFsFragment, createArtifactFs, toFs } from './artifacts' +import { Mutable, dedupe, isNonNullable, isWindows, makeRelative, memoize, wrapWithProxy } from './utils' +import { ModuleResolver } from './runtime/resolver' +import { createAuth, getAuth } from './auth' +import { CodeCache, createCodeCache } from './runtime/utils' +import { getFs } from './execution' +import { coerceToPointer, isDataPointer, pointerPrefix, toAbsolute } from './build-fs/pointers' +import { createNpmLikeCommandRunner } from './pm/publish' + +function getReplacementsForTarget(data: TargetsFile[string], target: string): Record { + const replacements: Record = {} + for (const [k, v] of Object.entries(data)) { + const binding = v[target] + if (binding) { + replacements[k] = binding + } + } + + return replacements +} + + +export interface LoaderOptions { + deployTarget?: string + backend?: { + } + outDir?: string + workingDirectory?: string + + generateExports?: boolean + exportSymbolData?: boolean + + // This will embed a logger into the app + loggerModule?: string + + beforeSynth?: terraform.SynthHook[] +} + +interface SynthLogger { + console: typeof console + stdout: typeof process.stdout + stderr: typeof process.stderr +} + +function createSynthLogger(p: typeof process, c: typeof console): SynthLogger { + const logEvent = getLogger().emitSynthLogEvent + const logMethods = { + log: (...args: any[]) => logEvent({ level: 'info', args }), + warn: (...args: any[]) => logEvent({ level: 'warn', args }), + error: (...args: any[]) => logEvent({ level: 'error', args }), + debug: (...args: any[]) => logEvent({ level: 'debug', args }), + trace: (...args: any[]) => logEvent({ level: 'trace', args }), + } + + const consoleWrap = wrapWithProxy(c, logMethods) + // TODO: wrap process + + return { console: consoleWrap, stdout: p.stdout, stderr: p.stderr } +} + +export function createrLoader( + artifactFs: BuildFsFragment, + targets: TargetsFile, + infraFiles: Record, + deployables: Record, + infraPointers: Record>, + packageResolver: PackageResolver, + moduleResolver: ModuleResolver, + buildTarget: BuildTarget, + deploymentId: string, + sourcemapParser: SourceMapParser, + options: LoaderOptions = {} +) { + const { + deployTarget, + workingDirectory = process.cwd(), + outDir = workingDirectory, + } = options + + const codeCache = createCodeCache(getFs(), getV8CacheDirectory()) + const proxyCache = new Map() + const reverseInfraFiles = Object.fromEntries(Object.entries(infraFiles).map(x => x.reverse())) as Record + + let getCredentials: ReturnType | undefined + function getCredentialsFn() { + return getCredentials ??= createGetCredentials(getAuth()) + } + + function createRuntime( + sources: { name: string, source: string }[], + getSource: (fileName: string, id: string, virtualLocation: string) => string, + solvePerms: (target: any, getContext: (target: any) => any, globals?: Record, args?: any[], thisArg?: any) => any, + ) { + const capturedSymbolData = new Map() + const onExitScope = options.exportSymbolData + ? (context: Context, val: any) => { capturedSymbolData.set(val, context) } + : undefined + + const context = createContextStorage({}, { onExitScope }, workingDirectory) + + function createTerraformState(): terraform.State { + return { + names: new Set(), + registered: new Map(), + backends: new Map(), + moved: [], + tables: new Map(), + serialized: new Map(), + serializers: new Map(), + secrets: new Map(), + } + } + + function createExportedSymbols(getSymbolId: (symbol: NonNullable) => number) { + function getExecutionScopeId(scopes: Scope[]) { + const symbols = scopes.slice(0, -1) + .map(x => x.symbol) + .filter(isNonNullable) + + return symbols.map(getSymbolId).join(':') + } + + const files: Record = {} + for (const [k, v] of capturedSymbolData.entries()) { + const scopes = v.scope2 + const assignmentSymbol = scopes[scopes.length - 1]?.assignmentSymbol + if (!assignmentSymbol) continue + + const id = getSymbolId(assignmentSymbol) + const executionScope = getExecutionScopeId(scopes) + + const exports = files[assignmentSymbol.fileName] ??= {} + const values = exports[executionScope] ??= { symbols: {}, context: v } + if (id in values.symbols) { + // throw new Error(`Found non-empty slot when assigning value to an execution scope`) + getLogger().log(`Found non-empty slot when assigning value to an execution scope: ${assignmentSymbol.name}`) + continue + } + + values.symbols[id] = k + } + + const Export = getExportClass() + + for (const [fileName, scopes] of Object.entries(files)) { + for (const [k, v] of Object.entries(scopes)) { + function createExport() { + new Export({ symbols: v.symbols }, { + id: `__exported-symbol-data`, + source: fileName, + executionScope: k, + } as any) + } + + context.restore(v.context, createExport) + } + } + } + + const terraformState = createTerraformState() + const tfGlobals =[ + () => terraformState, + (peek?: boolean) => peek ? context.getId(-1) : context.getId(), + () => context.getModuleId(), + () => context.getNamedContexts(), + () => context.getScopes(), + options.exportSymbolData ? createExportedSymbols : undefined, + ] as const + + const runtimeToSourceFile = new Map(sources.map(s => [s.name, s.source])) + const sourceToRuntimeFile = new Map(sources.map(s => [s.source, s.name])) + + const cache: Record = {} + const ctx = createVmContext() + + function createLinker() { + function getNodeModule(specifier: string, req = defaultRequire) { + switch (specifier) { + case 'node:path': + return wrapPath(tfModule.Fn, tfModule.internalState) + case 'node:module': + return patchModule(createRequire2) + + default: + return req(specifier) + } + } + + function createCjs(m: Module, opt?: ModuleCreateOptions) { + const specifier = m.name + if (m.typeHint === 'builtin') { + return createSyntheticCjsModule(() => getNodeModule(specifier)) + } + + const _ctx = opt?.context ?? ctx + + function createScript(text: string, name: string, cacheKey?: string) { + const req = createRequire2(m.id) + + return createScriptModule( + _ctx.vm, + text, + name, + req, + codeCache, + cacheKey, + sourcemapParser, + process, + req, + opt?.importModuleDynamically, + ) + } + + if (specifier.startsWith(pointerPrefix)) { + const pointer = coerceToPointer(!isDataPointer(specifier) && m.fileName?.startsWith(pointerPrefix) ? m.fileName : specifier) + const data = getSource(specifier, m.name, m.id) + if (typeof data !== 'string') { + createSyntheticCjsModule(() => data) + } + + return createScript(data, toAbsolute(pointer), pointer.hash) + } + + const fileName = m.fileName + if (!fileName) { + throw new Error(`No file name: ${m.name} [${m.id}]`) + } + + const extname = path.extname(fileName) + switch (extname) { + case '.mjs': + throw new Error(`Not expected: ${m.id}`) + case '.json': + return createSyntheticCjsModule(() => JSON.parse(getSource(fileName, specifier, m.id))) + case '.node': + return createSyntheticCjsModule(() => require(fileName)) + } + + const data = getSource(fileName, specifier, m.id) + + return createScript(data, fileName) + } + + async function createModuleWithReplacements(m: Module, symbols: Record, opt?: ModuleCreateOptions) { + // const data = getSource(m.fileName!, m.name, m.id) + + // ES6 modules are always in strict mode so it's unlikely to see "use strict" there + // const hint = m.typeHint ?? (data.startsWith('"use strict";') ? 'cjs' : undefined) + // if (hint === 'cjs') { + + // } + const mod = new vm.SyntheticModule(Object.keys(symbols), () => {}, { context: opt?.context?.vm ?? ctx.vm, identifier: m.name }) + await mod.link((() => {}) as any) + const proxy = wrapNamespace(m.id, mod.namespace) + const req = createRequire2(m.id, false) + for (const [k, v] of Object.entries(symbols)) { + defineProperty(proxy, k, { + enumerable: true, + configurable: false, + get: () => req(v.moduleSpecifier)[v.symbolName] + }) + } + + return mod + } + + function createEsm(m: Module, opt?: ModuleCreateOptions) { + const location = m.fileName! + const source = runtimeToSourceFile.get(location) + const moduleName = source ? getModuleName(source) : undefined + if (source) { + sourcemapParser.setAlias(infraFiles[location] ?? location, source) + } + + const replacementsData = targets[location] + if (replacementsData) { + getLogger().log(`Loaded symbol bindings for: ${m.id}`, Object.keys(replacementsData)) + + return createModuleWithReplacements(m, getReplacementsForTarget(replacementsData, deployTarget!), opt) + } + + const data = getSource(m.fileName!, m.name, m.id) + if (source) { + ;(m as Mutable).fileName = source // XXX: needed to make `import.meta.filename` align with CJS + } + const module = new vm.SourceTextModule(data, { + identifier: location, + context: opt?.context?.vm ?? ctx.vm, + importModuleDynamically: opt?.importModuleDynamically as any, + initializeImportMeta: opt?.initializeImportMeta, + }) + + if (moduleName) { + const evaluate = module.evaluate.bind(module) + module.evaluate = (opt) => { + return context.run({ moduleId: moduleName }, () => evaluate(opt)) + .then(() => { + if (options.generateExports && !!deployables[location]) { + createExportResource(location, moduleName, module.namespace, 'esm') + } + }) + } + } + + return module + } + + return createModuleLinker(getFs(), moduleResolver, { createCjs, createEsm }, ctx) + } + + const ctxTarget = deployTarget ?? 'local' + const getContext = (t: any) => { + const boundContexts = t[kBoundContext] as Record | undefined + if (!boundContexts) { + getLogger().log('perms: no bound context found, using current context') + + return context.get(ctxTarget)?.[0] + } + + return boundContexts[ctxTarget] + } + + const hostPermissions: any[] = [] + function solveHostPerms(target: any, args?: any[], thisArg?: any) { + const s = solvePerms(target, getContext, undefined, args, thisArg) + hostPermissions.push(...s) + } + + function getHostPerms() { + return dedupe(hostPermissions.flat(100)) + } + + const defaultRequire = createRequire2(workingDirectory) + const tfModule = defaultRequire('synapse:terraform')[unproxy] as typeof terraform + const constructsModule = tfModule.init(...tfGlobals) + + constructsModule.registerBackend('inmem', {}) + + if (options.beforeSynth) { + constructsModule.registerBeforeSynthHook(...options.beforeSynth) + } + + function getExportClass(require = defaultRequire) { + const { Export } = require('synapse:lib') as typeof import('synapse:lib') + + return Export + } + + function createAsset(target: string, importer: string) { + const resolvedImporter = runtimeToSourceFile.get(importer) ?? importer + const location = path.resolve(path.dirname(resolvedImporter), target) + + const { createFileAsset } = defaultRequire('synapse:lib') as typeof import('synapse:lib') + const asset = createFileAsset(location) + + return asset.pointer + } + + // Checks if the file will be deployed + function isInfra(fileName: string) { + const aliased = reverseInfraFiles[fileName] + if (aliased) { + return true + } + + return !!runtimeToSourceFile.has(fileName) + } + + // Deferred functions are executed after all modules have executed + const deferred: { ctx: Context, fn: () => void }[] = [] + function executeDeferred() { + for (const { ctx, fn } of deferred) { + context.restore(ctx, fn) + } + } + + function wrapNamespace(id: string, exports: any) { + if (isProxy(exports)) { + return exports + } + + const cached = proxyCache.get(exports) + if (cached) { + return cached + } + + const proxy = createModuleProxy(context, id, undefined, exports, false, undefined, solveHostPerms, true) + proxyCache.set(exports, proxy) + + return proxy + } + + function wrapExports(spec: string, virtualId: string | undefined, exports: any, isInfra: boolean = false) { + // if (isProxy(exports)) { + // return exports + // } + + const cached = proxyCache.get(exports) + if (cached) { + return cached + } + + if ( + isBuiltin(spec) || + spec.startsWith(pointerPrefix) || + spec.startsWith('synapse:') + ) { + const normalized = isBuiltin(spec) && !spec.startsWith('node:') ? `node:${spec}` : spec + const proxy = createModuleProxy(context, normalized, undefined, exports, isInfra, undefined, solveHostPerms) + proxyCache.set(exports, proxy) + + return proxy + } + + const proxy = createModuleProxy(context, spec, virtualId, exports, isInfra, packageResolver, solveHostPerms) + proxyCache.set(exports, proxy) + + return proxy + } + + function createVmContext() { + const _globalThis = createGlobalProxy(globalThis) as typeof globalThis + + // LOGGING + let logger: any + let loggerModule: any + function getRuntimeLogger() { + if (!options.loggerModule || logger === null) { + return + } + if (logger) { + return logger + } + + loggerModule = defaultRequire(options.loggerModule) + const Logger = loggerModule?.default?.loggerConstructor + const l = Logger ? new Logger('0', '0', deploymentId) : undefined + if (!l) { + logger = null + return + } + + return (logger = l) + } + + function getConsole(id: string, name?: string) { + const logger = getRuntimeLogger() + if (!logger) { + return _globalThis.console + } + + return loggerModule.createConsole(id, logger, name) + } + // END LOGGING + + function getPermissions(obj: any) { + // return solvePerms(obj, getContext, { + // console: logger ? loggerModule.createConsole('', logger) : undefined + // }) + + return solvePerms(obj, getContext) + } + + function getPointer(fileName: string, key: string) { + fileName = sourceToRuntimeFile.get(fileName) ?? fileName + const actualFile = infraFiles[fileName] ?? fileName + const filePointers = infraPointers[actualFile] + if (!filePointers) { + throw new Error(`No pointers found for file: ${fileName}`) + } + + const p = filePointers[key] + if (!p) { + throw new Error(`No pointer found for key "${key}" in ${fileName}`) + } + + return p + } + + const commandRunner = createNpmLikeCommandRunner(workingDirectory) + + registerSourceMapParser(sourcemapParser, Error) + + let wrappedJson: typeof JSON + function getWrappedJson() { + if (typeof tfModule === 'undefined') { + return JSON + } + return wrappedJson ??= createGlobalProxy({ JSON: wrapJSON(tfModule.Fn, tfModule.internalState) }).JSON + } + + const wrapped = createSynthLogger(process, _globalThis.console) + const ctx = vm.createContext({ + Promise, + URL, + Set, + Blob, + Map, + Symbol, + Buffer, + Object, + Uint8Array, + ArrayBuffer, + Error, + Response, + setTimeout, + clearTimeout, + get JSON() { + return getWrappedJson() + }, + RegExp: _globalThis.RegExp, + // console: _globalThis.console, + console: wrapped.console, + globalThis: _globalThis, // Probably shouldn't do this + get __getCredentials() { + return getCredentialsFn() + }, + __getConsole: getConsole, + // __getBackendClient: getBackendClient, + __getContext: () => context, + __createAsset: createAsset, + __getBuildDirectory: () => outDir, + __getPermissions: getPermissions, + __getLogger: getLogger, + __getArtifactFs: () => artifactFs, + __defer: (fn: () => void) => { + const ctx = context.save() + deferred.push({ ctx, fn }) + }, + __cwd: () => workingDirectory, + __buildTarget: buildTarget, + __deployTarget: deployTarget, + __runCommand: commandRunner, + __getCurrentId: () => context.getScope(), + __getPointer: getPointer, + __getCallerModuleId: context.getModuleId, + __requireSecret: (envVar: string, type: string) => constructsModule.registerSecret(envVar, type), + __scope__: context.run, + __wrapExports: wrapExports, + }) + + const globals = vm.runInContext('this', ctx) + patchBind(globals.Function, globals.Object) + + return { + globals: _globalThis, + vm: ctx, + } + } + + function createRequire2(importer: string, addWrap = true) { + const getNodeRequire = () => { + if (importer.startsWith(pointerPrefix)) { + return createRequire(bootstrapPath) + } + + const resolved = moduleResolver.getFilePath(importer) + + return createRequire(resolved.startsWith(pointerPrefix) ? bootstrapPath : resolved) + } + + return function require2(spec: string) { + function addSymbol(exports: any, virtualId?: string, isInfra = false) { + if (!addWrap) { + return exports + } + return wrapExports(spec, virtualId, exports, isInfra) + } + + if (isBuiltin(spec)) { + if (spec === 'buffer' || spec === 'node:buffer') { + // Don't add the proxy here, otherwise we have to deal with `instanceof` not working + // correctly for `Buffer` with built-in modules + return getNodeRequire()(spec) + } + if (spec === 'path' || spec === 'node:path') { + return addSymbol(wrapPath(tfModule.Fn, tfModule.internalState)) + } + if (spec === 'module' || spec === 'node:module') { + return addSymbol(patchModule(createRequire2)) + } + return addSymbol(getNodeRequire()(spec)) + } + + // XXX + // if (spec === 'monaco-editor') { + // cache[spec] ??= { exports: createFakeModule(spec, context) } + // return cache[spec].exports + // } + + const location = moduleResolver.resolveVirtual(spec, importer) + if (cache[location] !== undefined) { + return cache[location].exports + } + + const locationIsPointer = isDataPointer(location) + + const physicalLocation = locationIsPointer + ? toAbsolute(location) + : moduleResolver.getFilePath(location).toString() // XXX: `toString` to convert pointers to primitives + + const cacheKey = locationIsPointer ? location.hash : undefined + const extname = path.extname(physicalLocation) + if (extname === '.node') { + cache[location] = { exports: addSymbol(getNodeRequire()(physicalLocation), location) } + return cache[location].exports + } else if (extname === '.json') { + cache[location] = { exports: JSON.parse(getSource(physicalLocation, spec, location)) } + return cache[location].exports + } + + // ~100ms on `example` + const text = getSource(physicalLocation, spec, location) + + if (typeof text !== 'string') { + cache[location] = { exports: addSymbol(text, location) } + return cache[location].exports + } + + // Without the `synapse:` check then we would serialize things like `cloud.getArtifactFs` incorrectly + const isInfraSource = isInfra(physicalLocation) && !spec.startsWith('synapse:') + const replacementsData = targets[physicalLocation] + if (replacementsData) { + getLogger().log(`Loaded symbol bindings for: ${physicalLocation}`, Object.keys(replacementsData)) + } + + function createModuleStub(val: any) { + if (!replacementsData || !deployTarget) { + return val + } + + const targets = getReplacementsForTarget(replacementsData, deployTarget) + const desc = { + ...Object.getOwnPropertyDescriptors(val), + ...Object.fromEntries( + Object.entries(targets).map(([k, v]) => { + return [k, { get: () => require2(v.moduleSpecifier)[v.symbolName], enumerable: true }] as const + }) + ) + } + + return Object.create(null, desc) + } + + const require2 = createRequire2(location) + let wrapped = addSymbol(createModuleStub({}), location, isInfraSource) + const module_ = { + get exports() { + return wrapped + }, + set exports(newVal) { + wrapped = addSymbol(createModuleStub(newVal), location, isInfraSource) + } + } + + cache[location] = module_ + + return runText(text, physicalLocation, module_, require2, cacheKey) + } + } + + function getModuleName(targetModule: string) { + const moduleName = path.relative(buildTarget.workingDirectory, targetModule) + if (moduleName.startsWith('..')) { + throw new Error(`Module is located outside of the current working directory: ${targetModule}`) + } + if (isWindows()) { + return moduleName.replaceAll('\\', '/') + } + return moduleName + } + + function runText( + text: string, + location: string, + module: { exports: {} }, + require2 = createRequire2(location), + cacheKey?: string + ) { + const source = runtimeToSourceFile.get(location) + const moduleName = source ? getModuleName(source) : undefined + if (source) { + sourcemapParser.setAlias(infraFiles[location] ?? location, source) + } + + try { + const cjs = createScriptModule(ctx.vm, text, source ?? location, require2, codeCache, cacheKey, sourcemapParser, process, undefined, undefined, module) + + runTask( + 'require', + path.relative(buildTarget.workingDirectory, location), + () => { + if (moduleName) { + return context.run({ moduleId: moduleName }, cjs.evaluate) + } + + return cjs.evaluate() + }, + 25 + ) + } catch (e) { + getLogger().log('failed running file', location) + throw e + } + + if (options.generateExports && !!moduleName && !!deployables[location]) { + const _exports = (module.exports as any)[unproxy] + createExportResource(location, moduleName, _exports, 'cjs') + } + + return module.exports + } + + function createExportResource(location: string, moduleName: string, exports: any, type: 'cjs' | 'esm') { + const Export = getExportClass() + const opt = { + isModule: true, + source: moduleName, + id: makeRelative(buildTarget.workingDirectory, location).replace(/\.js$/, ''), + publishName: makeRelative(workingDirectory, location).replace(/\.infra\.js$/, '.js'), + } + + const exportObj = type === 'esm' || (typeof exports === 'object' && exports?.__esModule) + ? { __esModule: true, ...exports } + : exports + + context.run({ moduleId: moduleName }, () => new Export(exportObj, opt)) + } + + function getCore() { + return defaultRequire('synapse:core') as typeof import('synapse:core') + } + + function getSrl() { + return defaultRequire('synapse:srl') as typeof import('synapse:srl') + } + + return { createRequire2, runText, executeDeferred, constructsModule, getHostPerms, getCore, getSrl, getContext: () => context, createLinker, globals: ctx.globals } + } + + const bootstrapName = '#bootstrap.js' + const bootstrapPath = path.resolve(outDir, bootstrapName) + const providerParams = { + buildDirectory: path.relative(workingDirectory, buildTarget.buildDir), + outputDirectory: path.relative(workingDirectory, outDir), + } + + function handleError(e: unknown): never { + if (!(e instanceof Error && 'serializeStack' in e)) { + throw e + } + + const serializeStack = e.serializeStack as any[] + const top = serializeStack[0] + if (!top) { + throw e + } + + const capturedSet = new Set(serializeStack) + const visited = new Set() + + const indent = (s: string, depth: number) => `${' '.repeat(depth)}${s}` + function printCapturedObject(o: any, depth = 0): string { + if (visited.has(o)) { + return indent(``, depth) // TODO: use the rendered location + } + + visited.add(o) + + if (typeof o === 'function' && moveable in o) { + const desc = o[moveable]() + const module = desc.module + const location = module ? getArtifactOriginalLocation(module) : undefined + if (!module || !location) { + return indent(require('node:util').inspect(desc), depth) + } + + const name = o.name || '' + + return `${indent(`${name} at ${location}`, depth)}` + + // return [ + // `${indent(`${name} at ${location}`, depth)}`, + // ...captured.filter(x => capturedSet.has(x)).map(x => printCapturedObject(x, depth)) + // ].join('\n') + } + + return indent(require('node:util').inspect(o), depth) + } + + // const s = m.map(y => typeof y === 'string' ? y : require('node:util').inspect(y)) + const newE = new Error((e as any).message, e.cause ? { cause: e.cause } : undefined) + newE.stack = [ + `${e.name}: ${e.message}`, + ...serializeStack.filter(x => typeof x === 'function').reverse().map(x => printCapturedObject(x, 1)) + ].join('\n') + + throw newE + } + + function synth( + entrypoints: string[], + runtime: ReturnType + ) { + const coreProvider = new (runtime.getCore()).Provider(providerParams as any) + const targetProvider = new (runtime.getSrl()).Provider('#default') + const req = runtime.createRequire2(bootstrapPath) + + runTask('run', 'bootstrap', () => { + runtime.getContext().run({ contexts: [coreProvider, targetProvider] }, () => { + for (const fileName of entrypoints) { + req(fileName) + } + }) + + runtime.executeDeferred() + }) + + try { + const terraform = runTask('emit', 'synth', () => runtime.constructsModule.emitTerraformJson(), 1) + + return { terraform, permissions: runtime.getHostPerms() } + } catch (e) { + handleError(e) + } + } + + // TEMPORARY + async function synthEsm( + entrypoints: string[], + runtime: ReturnType + ) { + const coreProvider = new (runtime.getCore()).Provider(providerParams as any) + const targetProvider = new (runtime.getSrl()).Provider('#default') + const linker = runtime.createLinker() + + await runTask('run', 'bootstrap', async () => { + await runtime.getContext().run({ contexts: [coreProvider, targetProvider] }, async () => { + for (const fileName of entrypoints) { + const esm = await linker.getEsm(process.cwd(), fileName) + await linker.evaluateEsm(esm) + } + }) + + runtime.executeDeferred() + }) + + try { + const terraform = runTask('emit', 'synth', () => runtime.constructsModule.emitTerraformJson(), 50) + + return { terraform, permissions: runtime.getHostPerms() } + } catch (e) { + handleError(e) + } + } + + return { synth, createRuntime, synthEsm } +} + +function wrapJSON(Fn: typeof terraform['Fn'], internalState: typeof terraform.internalState): typeof JSON { + const parse: typeof JSON['parse'] = (text, reviver) => { + if (typeof text !== 'function' || !(internalState in text)) { + return JSON.parse(text, reviver) + } + + if (reviver) { + throw new Error(`Using "reviver" is not implemented when decoding resources`) + } + + return Fn.jsondecode(text) + } + + const stringify: typeof JSON['stringify'] = (value, replacer, space) => { + // TODO: implement this by recursively checking if the value contains a resource + // If so, we need to call `Fn.jsonencode` + + return JSON.stringify(value, replacer as any, space) + } + + return { + parse, + stringify, + [Symbol.toStringTag]: 'JSON', + } +} + +function wrapPath(Fn: typeof terraform['Fn'], internalState: typeof terraform.internalState) { + const exports = { ...require('path') as typeof import('path') } + + const originalDirname = exports.dirname + const originalBasename = exports.basename + const originalRelative = exports.relative + const originalResolve = exports.resolve + + exports.dirname = function dirname(path: string) { + if (typeof path === 'function' && internalState in path) { + return Fn.dirname(path) + } + + return originalDirname(path) + } + + exports.basename = function basename(path: string) { + if (typeof path === 'function' && internalState in path) { + return Fn.basename(path) + } + + if (isDataPointer(path)) { + return path.hash + } + + return originalBasename(path) + } + + exports.relative = function relative(from: string, to: string) { + if ((typeof from === 'function' && internalState in from) || (typeof to === 'function' && internalState in to) ) { + return Fn.trimprefix(Fn.replace(to, from, ''), '/') + } + + return originalRelative(from, to) + } + + exports.resolve = function resolve(...segments: string[]) { + if (segments.some(s => typeof s === 'function' && internalState in s)) { + return process.cwd() + '/' + segments[segments.length - 1] // XXX: not correct + } + + return originalResolve(...segments) + } + + + return exports +} + +function patchModule(createRequire2: (origin: string) => (id: string) => any): typeof import('module') { + const exports = require('module') as typeof import('module') + + return new Proxy(exports, { + get: (_, prop) => { + if (prop === 'createRequire') { + return function createRequire(origin: string) { + const req = exports.createRequire(origin) + + return Object.assign(createRequire2(origin), { + resolve: (id: string) => req.resolve(id) + }) + } + } + + return exports[prop as keyof typeof exports] + } + }) +} + + +function isObjectOrNullPrototype(proto: any) { + return proto === Object.prototype || proto === null || proto.constructor?.name === 'Object' +} + +function shouldTrap(value: any): boolean { + if (typeof value === 'function') { + // PERF: this hurts perf by ~10% + // But it's needed to make sure resources aren't instantiated outside of their context + if (moveable in value) { + return false + } + + return true + } + if (Reflect.getOwnPropertyDescriptor(value, moveable2)) { + return false + } + // value instanceof stream.Readable + if (value instanceof Buffer || value.constructor === Uint8Array || value.constructor === ArrayBuffer) { + return false + } + // if (value.constructor === Promise || value instanceof Error) { + // return false + // } + + // If the value is directly serializable then we do not need to trap it + const proto = Object.getPrototypeOf(value) + if (!isObjectOrNullPrototype(proto)) { + return true + } + + const descriptors = Object.getOwnPropertyDescriptors(value) + for (const desc of Object.values(descriptors)) { + if (desc.get || desc.set || _shouldTrap(desc.value)) { + return true + } + } + + return false +} + +// Maybe support `CoffeeScript` +// https://github.com/jashkenas/coffeescript/blob/main/lib/coffeescript/register.js +// +// The repo is not very active though + +function createFakeModule(id: string, ctx: ContextStorage) { + let trappedProto = false + + function createProxy(operations: any[] = [], parent?: any): any { + const getOps = () => { + return { + valueType: 'reflection', + operations: [ + { type: 'import', module: id }, + ...operations + ], + } + } + + const cache = new Map() + + const inner = function() {} + const p: any = new Proxy(inner, { + get: (_, prop) => { + if (prop === unproxy) { + return inner + } else if (prop === unproxyParent) { + return parent + } else if (prop === moveable2) { + return getOps + } + + if (typeof prop === 'symbol') { + return (inner as any)[prop] + } + + if (cache.has(prop)) { + return cache.get(prop) + } + + const v = createProxy([...operations, { type: 'get', property: prop }], p) + cache.set(prop, v) + + return v + }, + apply: () => { + throw new Error(`Cannot use module "${id}" in this runtime`) + }, + construct: (target, args, newTarget) => { + const _target = unwrapProxy(target) + const _newTarget = unwrapProxy(newTarget) + + const result = createProxy( + [...operations, { + type: 'construct', + args, + }], + p, + ) + + // XXX: bind context to the instance if `permissions` exists + const ctor = _newTarget ?? _target + if (Symbol.for('permissions') in ctor) { + const contexts = Object.fromEntries(Object.entries(ctx.getNamedContexts()).map(([k, v]) => [k, v[0]])) + Object.assign(result, { [kBoundContext]: contexts }) + } + + return result + }, + has: (target, prop) => { + if (prop === moveable2 || prop === unproxy || prop === unproxyParent) { + return true + } + + return Reflect.has(target, prop) + }, + // ownKeys: (target) => { + // const keys = Reflect.ownKeys(target) + // if (!keys.includes(moveable2)) { + // return [moveable2, ...keys] + // } + // return keys + // }, + getPrototypeOf: () => !trappedProto ? (trappedProto = true, createProxy(operations)) : null, + getOwnPropertyDescriptor: (target, prop) => { + if (prop === moveable2) { + return { configurable: true, value: getOps } + } + + return Reflect.getOwnPropertyDescriptor(target, prop) + }, + }) + + return p + } + + return createProxy() +} + +// Any module that we have not compiled is considered "external" +export const moveable = Symbol.for('__moveable__') +function createModuleProxy(ctx: ContextStorage, spec: string, virtualId: string | undefined, value: any, isInfra: boolean, resolver?: PackageResolver, solvePerms?: (target: any, args?: any[]) => any, isNamespace?: boolean) { + return createSerializationProxy(value, [{ + type: 'import', + module: spec, + virtualId, + }], undefined, isInfra, undefined, ctx, resolver, solvePerms, isNamespace) +} + +function createGlobalProxy(value: any) { + return createSerializationProxy(value, [{ type: 'global' }]) +} + +export function isProxy(o: any, checkPrototype = false) { + return !!o && ((checkPrototype && unproxy in o) || !!Reflect.getOwnPropertyDescriptor(o, moveable2)) +} + +export function unwrapProxy(o: any, checkPrototype = false): any { + if (isProxy(o, checkPrototype)) { + return unwrapProxy(o[unproxy]) + } + + return o +} + +const c = new WeakMap() +function _shouldTrap(target: any) { + if (target === null || (typeof target !== 'function' && typeof target !== 'object')) { + return false + } + + const f = c.get(target) + if (f !== undefined) { + return f + } + + const v = shouldTrap(target) + c.set(target, v) + + return v +} + +// `isInfra` can probably be removed + +const unproxy = Symbol.for('unproxy') +const moveable2 = Symbol.for('__moveable__2') +const unproxyParent = Symbol.for('unproxyParent') +const kBoundContext = Symbol.for('boundContext') +const kSourceModule = Symbol.for('kSourceModule') +const reflectionType = Symbol.for('reflectionType') + +// This is an issue with React when capturing JSX elements: +// TypeError: 'ownKeys' on proxy: trap returned extra keys but proxy target is non-extensible +// +// Not really a problem because you're technically not supposed to extract out raw fragments + + +// Need to do this indirectly because we cannot always trap `defineProperty` +const allDescriptors = new Map>() +function defineProperty(proxy: any, key: PropertyKey, descriptor: PropertyDescriptor) { + const descriptors = allDescriptors.get(proxy) + if (!descriptors) { + return false + } + + descriptors.set(key, descriptor) + return true +} + +function createSerializationProxy( + value: any, + operations: any[], + parent?: any, + isInfra = false, + isProto = false, + ctx?: ContextStorage, + resolver?: PackageResolver, + solvePerms?: (target: any, args?: any[], thisArg?: any) => any, + isModuleNamespace = false, +): any { + const cache = new Map() + const descriptors = isModuleNamespace ? new Map() : undefined + + const extraHandlers = parent === undefined ? { + getPrototypeOf: (target: any) => { + // Only trap modules once + if (operations.length > 1 || isProto) { + return Reflect.getPrototypeOf(target) + } + + return createSerializationProxy(value, operations, parent, isInfra, true, ctx, resolver, solvePerms) + }, + } : undefined + + let expanded: any + function expand() { + return expanded ??= { + valueType: 'object', + properties: { ...value }, + } + } + + function resolveReflection(moduleIdOverride?: string) { + const op = operations[0] + if (op.type !== 'import') { + return { + valueType: 'reflection', + operations, + } + } + + if (moduleIdOverride) { + return { + valueType: 'reflection', + operations: [ + { + type: 'import', + ...op, + module: moduleIdOverride, + location: undefined, + }, + ...operations.slice(1) + ], + } + } + + if (!resolver) { + return { + valueType: 'reflection', + operations, + } + } + + const { module } = resolver.reverseLookup(op.module, op.location, op.virtualId) + + return { + valueType: 'reflection', + operations: [ + { + type: 'import', + module, + // These cause too many perf issues atm + // packageInfo, + // dependencies, + }, + ...operations.slice(1) + ], + } + } + + function getMoveableDescFn() { + // This is currently used for the generated Terraform classes + const moduleOverride = parent?.[Symbol.for('moduleIdOverride')] ?? value?.[Symbol.for('moduleIdOverride')] + // TODO: is this still needed (yes it is) + // Runtime failures happen without it, e.g. `throw __scope__("Error", () => new Error("No target set!"));` + if (isInfra && !moduleOverride) { + cache.set(moveable2, expand) + cache.set(reflectionType, undefined) + return expand + } + + const fn = () => resolveReflection(moduleOverride) + cache.set(moveable2, fn) + cache.set(reflectionType, operations[0].type) + + return fn + } + + const moveableDesc = { + configurable: true, + get: getMoveableDescFn, + } + + const p: any = new Proxy(value, { + ...extraHandlers, + get: (target, prop, recv) => { + if (cache.has(prop)) { + return cache.get(prop)! + } + + if (prop === unproxy) { + return value + } else if (prop === unproxyParent) { + return parent + } else if (prop === kSourceModule) { + return operations[0].module + } else if (prop === reflectionType) { + const moduleOverride = parent?.[Symbol.for('moduleIdOverride')] ?? value?.[Symbol.for('moduleIdOverride')] + if (isInfra && !moduleOverride) { + cache.set(moveable2, expand) + cache.set(reflectionType, undefined) + return undefined + } + + const fn = () => resolveReflection(moduleOverride) + cache.set(moveable2, fn) + cache.set(reflectionType, operations[0].type) + + return operations[0].type + } else if (prop === moveable2) { + return getMoveableDescFn() + } else if (descriptors?.has(prop)) { + const desc = descriptors.get(prop)! + const result = desc.get ? desc.get() : desc.value + if (!_shouldTrap(result)) { + return result + } + + const sub = createSerializationProxy( + result, + [...operations, { + type: 'get', + property: prop, + }], + p, + isInfra, + isProto, + ctx, + resolver, + solvePerms, + ) + cache.set(prop, sub) + + return sub + } + + const result = target[prop] + // Needed to avoid an infinite loop from the AWS SDK + if (prop === 'valueOf') { + return result + } + + // Only capture bind if we're referencing `bind` on `Function` + // if (prop === 'bind' && result === functionBind) { + // const bind: typeof functionBind = function bind(thisArg, ...args) { + + // } + // cache.set(prop, bind) + + // return bind + // } + + // Checking the descriptor is needed otherwise an error is thrown on non-proxyable descriptors + const desc = Reflect.getOwnPropertyDescriptor(target, prop) + if (typeof prop === 'symbol' || (desc?.configurable === false && desc.writable === false) || !_shouldTrap(result)) { + return result + } + + const sub = createSerializationProxy( + result, + [...operations, { + type: 'get', + property: prop, + }], + p, + isInfra, + isProto, + ctx, + resolver, + solvePerms, + ) + cache.set(prop, sub) + + return sub + }, + apply: (target, thisArg, args) => { + const result = Reflect.apply(target, unwrapProxy(thisArg, true), args) + if (!_shouldTrap(result)) { + return result + } + + // XXX: never trap the fn passed to `__scope__` + if (operations.length === 2 && operations[0].module === 'synapse:core' && operations[1].property === 'scope') { + return result + } + + return createSerializationProxy( + result, + [...operations, { + type: 'apply', + args, + thisArg, + }], + p, + isInfra, + isProto, + ctx, + resolver, + solvePerms, + ) + }, + construct: (target, args, newTarget) => { + // Assumption: `target` === `newTarget` + const _target = unwrapProxy(target) + const _newTarget = unwrapProxy(newTarget) + + const result = Reflect.construct(_target, args, _newTarget) + + // XXX: bind context to the instance if `permissions` exists + const ctor = _newTarget ?? _target + if (result && typeof result === 'object' && Symbol.for('permissions') in ctor) { + const contexts = Object.fromEntries(Object.entries(ctx?.getNamedContexts() ?? {}).map(([k, v]) => [k, v[0]])) + Object.assign(result, { [kBoundContext]: contexts }) + + const cm = ctor[Symbol.for('permissions')].$constructor + if (cm !== undefined) { + solvePerms?.(ctor, args, result) + } + } + + // if (!shouldTrap(result)) { + // return result + // } + + return createSerializationProxy( + result, + [...operations, { + type: 'construct', + args, + }], + p, + isInfra, + isProto, + ctx, + resolver, + solvePerms, + ) + }, + has: (target, prop) => { + if (prop === moveable2 || prop === unproxy || prop === unproxyParent) { + return true + } + + return Reflect.has(target, prop) + }, + set: (target, prop, newValue, recv) => { + cache.delete(prop) + + return Reflect.set(target, prop, newValue, recv) + }, + // ownKeys: (target) => { + // const keys = Reflect.ownKeys(target) + // if (!keys.includes(moveable2)) { + // return [moveable2, ...keys] + // } + // return keys + // }, + getOwnPropertyDescriptor: (target, prop) => { + if (prop === moveable2) { + return moveableDesc + } + + return Reflect.getOwnPropertyDescriptor(target, prop) + }, + // TODO: deleteProperty + }) + + if (descriptors) { + allDescriptors.set(p, descriptors) + } + + return p +} + +function createLazyEvalModule(fn: () => any) { + let val: any = undefined + let didInit = false + const state: any = {} + const init = () => { + didInit = true + val = fn() + + // Note: we are not preserving the order of operations here + // this could potentially cause issues + Object.setPrototypeOf(val, Object.getPrototypeOf(state)) + Object.defineProperties(val, Object.getOwnPropertyDescriptors(state)) + + return val + } + + return new Proxy(state, { + get: (_, prop) => { + if (didInit) { + return val[prop] + } + + if (prop in state) { + return state[prop] + } + + return init()[prop] + }, + has: (_, prop) => { + if (didInit) { + return prop in val + } + + if (prop in state) { + return true + } + + return prop in init() + }, + ownKeys: () => { + if (didInit) { + return Reflect.ownKeys(val) + } + + return Reflect.ownKeys(init()) + }, + getOwnPropertyDescriptor: (_, prop) => { + if (didInit) { + return Object.getOwnPropertyDescriptor(val, prop) + } + + if (Object.hasOwn(state, prop)) { + return Object.getOwnPropertyDescriptor(state, prop) + } + + return Object.getOwnPropertyDescriptor(init(), prop) + }, + setPrototypeOf: (_, v) => { + if (didInit) { + return Object.setPrototypeOf(val, v) + } + + return Object.setPrototypeOf(state, v) + }, + apply: (_, thisArg, args) => { + if (didInit) { + return Reflect.apply(val, thisArg, args) + } + + return Reflect.apply(fn(), thisArg, args) + }, + + }) +} + +function patchBind(FunctionCtor: typeof Function, ObjectCtor: typeof Object) { + const original = FunctionCtor.prototype.bind + FunctionCtor.prototype.bind = function (thisArg, ...args) { + const m = ObjectCtor.getOwnPropertyDescriptor(this, moveable)?.value + const bound = original.apply(this, arguments as any) + if (!m) { + return bound + } + + bound[moveable] = () => { + const desc = m() + if (desc.valueType === 'function') { + return { + valueType: 'bound-function', + boundTarget: this, + boundArgs: args, + boundThisArg: thisArg, + } + } + + return { + valueType: 'bound-function', + boundTarget: desc.boundTarget, + boundArgs: [...desc.boundArgs, ...args], + boundThisArg: desc.boundThisArg, + } + } + + return bound + } + + return { dispose: () => void (FunctionCtor.prototype.bind = original) } +} + +interface FsContext { + readonly workingDirectory: string +} + +interface Context { + readonly moduleId?: string + readonly scope: string[] + readonly symbols: terraform.Symbol[] + readonly scope2: Scope[] + readonly namedContexts: Record +} + +const kContext = Symbol.for('context') +const kContextType = Symbol.for('contextType') + +function getContextType(o: any, visited = new Set()): string | undefined { + if (!o || (typeof o !== 'object' && typeof o !== 'function') || visited.has(o)) { + return + } + + // FIXME + if (visited.size > 10) { + return + } + + const unproxied = unwrapProxy(o) + visited.add(unproxied) + + return o[kContextType] ?? getContextType(unproxied.constructor, visited) +} + +interface ContextHooks { + onExitScope?: (context: Context, result: any) => void +} + +type ContextStorage = ReturnType +function createContextStorage( + initialContext: Context['namedContexts'] = {}, + hooks: ContextHooks = {}, + workingDir: string +) { + const storage = new AsyncLocalStorage() + const getNamedContexts = () => { + const ctx = storage.getStore()?.namedContexts ?? initialContext + + return { ...ctx } + } + + function mergeContexts(left: any, right: any) { + for (const [k, v] of Object.entries(right)) { + // This is reversed so we pick the newest context first + left[k] = [v, ...(left[k] ?? [])] + } + } + + function run(scope: Scope, fn: (...args: T) => U, ...args: T): U { + const oldStore = storage.getStore() + + const namedContexts = { ...getNamedContexts() } + if (scope.contexts !== undefined) { + for (const ctx of scope.contexts) { + const contextType = getContextType(ctx) + const contextContribution = ctx[kContext] + if (!contextType && !contextContribution) { + throw new Error(`Object does not contribute a context: ${ctx}`) + } + + if (contextType) { + mergeContexts(namedContexts, { [contextType]: ctx }) + } + if (contextContribution) { + for (const ctx2 of contextContribution) { + const contextType = getContextType(ctx2) + if (!contextType) { + throw new Error(`Object does not contribute a context: ${ctx2}`) + } + mergeContexts(namedContexts, { [contextType]: ctx2 }) + } + } + } + } + + const symbols = oldStore?.symbols ?? [] + const scope2 = [...(oldStore?.scope2 ?? []), scope] + const context: Context = { + scope2, + namedContexts, + moduleId: scope.moduleId ?? oldStore?.moduleId, + symbols: scope.symbol ? [...symbols, scope.symbol] : symbols, + scope: [...(oldStore?.scope ?? []), scope.name ?? ''], + } + + const result = storage.run(context, fn, ...args) + + if (hooks.onExitScope) { + const handler = hooks.onExitScope + if (util.types.isPromise(result)) { + result.then(v => handler(context, v)) + } else { + handler(context, result) + } + } + + return result + } + + function getScope() { + const id = getId() + if (!id) { + throw new Error('Not within a scope') + } + + return id + } + + function getId(pos?: number) { + const scope = storage.getStore()?.scope.filter(x => !!x) + const sliced = pos && scope ? scope.slice(0, pos) : scope + if (!sliced || sliced.length === 0) { + return '' + } + + const id = sliced.join('--') + const moduleId = getModuleId() + + return moduleId ? `${moduleId.replace(/\//g, '--').replace(/\.(.*)$/, '')}_${id}` : id + } + + // TODO: this should ideally be apart of `scope` + function getModuleId() { + return storage.getStore()?.moduleId + } + + function mapSymbol(sym: NonNullable): NonNullable { + return { + ...sym, + fileName: path.relative(workingDir, sym.fileName), + } + } + + function getScopes() { + const scopes = storage.getStore()?.scope2 + if (!scopes) { + return [] + } + + return scopes.map(s => ({ + isNewExpression: s.isNewExpression, + callSite: s.symbol ? mapSymbol(s.symbol) : undefined, + assignment: s.assignmentSymbol ? mapSymbol(s.assignmentSymbol) : undefined, + namespace: s.namespace?.map(mapSymbol), + })) + } + + function get(type: keyof Context['namedContexts']) { + return getNamedContexts()[type] + } + + function save(): Context { + const ctx = storage.getStore() + if (!ctx) { + throw new Error('Not within a context') + } + + return ctx + } + + function restore(ctx: Context, fn: () => T): T { + return storage.run(ctx, fn) + } + + return { + save, + restore, + + run, + get, + getId, + getScope, + getScopes, + getModuleId, + getNamedContexts, + } +} + + +// Apple Dev Id +// W1337179404 +// 53JRUGDHXZ +// https://medium.com/anchore-engineering/developers-need-to-handle-macos-binary-signing-how-we-automated-the-solution-part-1-4433b32ae311 + +// { +// "main": "/path/to/bundled/script.js", +// "output": "/path/to/write/the/generated/blob.blob", +// "disableExperimentalSEAWarning": true, // Default: false +// "useSnapshot": false, // Default: false +// "useCodeCache": true // Default: false +// } + +// #include +// #include + +// static int rcs_addr_handle = 0; + +// std::pair get_resource( const char* sec_name ) +// { +// // get image header from a global address +// Dl_info img_info = {}; +// int img_index = dladdr( &rcs_addr_handle, &img_info ); +// if ( img_index == 0 ) return { nullptr, 0 }; +// auto* header = (struct mach_header_64*) img_info.dli_fbase; + +// // get resource address +// unsigned long size = 0; +// auto* addr = getsectiondata( header, "my_rcs_segment", sec_name, &size ); +// return { addr, size }; +// } diff --git a/src/logging.ts b/src/logging.ts new file mode 100644 index 0000000..e8b6dd0 --- /dev/null +++ b/src/logging.ts @@ -0,0 +1,377 @@ +import * as perf from 'node:perf_hooks' +import { Event, EventEmitter, addMetaListener, createEvent, createEventEmitter } from './events' +import type { BuildTarget } from './workspaces' +import type { ParsedPlan } from './deploy/deployment' +import type { TfState } from './deploy/state' +import type { ResolvedProgramConfig } from './compiler/config' +import { getBuildTarget, getBuildTargetOrThrow, getExecutionId, isInContext } from './execution' +import { InstallLifecycleEvent, PackageProgressEvent } from './cli/views/install' +import { memoize } from './utils' + +export interface PerfDetail { + readonly taskType: string + readonly taskName: string + readonly slowThreshold?: number + readonly aggregate?: boolean +} + +const markCounts: Record = {} +export function runTask(type: string, name: string, task: () => T, slowThreshold?: number, aggregate?: boolean): T { + const markName = `${type}-${name}` + const detail: PerfDetail = { + taskType: type, + taskName: name, + slowThreshold, + aggregate, + } + + const counts = markCounts[markName] = (markCounts[markName] ?? 0) + 1 + const id = `${markName}-${counts}` + perf.performance.mark(id, { detail }) + + const done = () => { + perf.performance.measure(`${id}-end`, { start: id, detail }) + perf.performance.clearMarks(id) + } + + const res = task() + if (res instanceof Promise) { + return res.finally(done) as any + } + + done() + + return res +} + +function createObserver(emit: OutputEventTrigger) { + const aggregates: Record = {} + const obs = new perf.PerformanceObserver(items => { + const measurements = items.getEntriesByType('measure') + for (const m of measurements) { + const detail = m.detail as PerfDetail + if (detail.aggregate) { + aggregates[detail.taskType] = (aggregates[detail.taskType] ?? 0) + m.duration + } else { + emit({ duration: m.duration, ...detail }) + } + } + + perf.performance.clearMeasures() + }) + + process.on('exit', () => { + for (const [k, v] of Object.entries(aggregates)) { + emit({ duration: v, taskType: k, taskName: '', slowThreshold: 0 }) + } + }) + + obs.observe({ entryTypes: ['measure'] }) + + return obs +} + +export interface BaseOutputMessage { + readonly type: string + readonly timestamp: Date + readonly context?: OutputContext + + // readonly system?: string +} + +export interface CommandEvent extends BaseOutputMessage { + readonly type: 'command' + readonly action: string + readonly status: 'started' | 'succeeded' | 'failed' + readonly context: OutputContext +} + +// Generic log event +export interface LogEvent extends BaseOutputMessage { + readonly type: 'log' + readonly level: 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'raw' + readonly args: any[] +} + +export interface PerfEvent extends BaseOutputMessage { + readonly type: 'perf' + readonly taskType: string + readonly taskName: string + readonly duration: number + readonly slowThreshold?: number +} + +export interface OutputContext { + readonly buildTarget: BuildTarget + readonly executionId?: string +} + +export function getOutputContext(): OutputContext | undefined { + if (!isInContext()) { + return + } + + const buildTarget = getBuildTarget() + if (!buildTarget) { + return + } + + return { + buildTarget, + executionId: getExecutionId(), + } +} + +// COMMON + +export interface ResolveConfigEvent extends BaseOutputMessage { + readonly type: 'resolve-config' + readonly config: ResolvedProgramConfig +} + +// DEPLOY + +export type DeployStatus = 'refreshing' | 'pending' | 'applying' | 'complete' | 'failed' +export type DeployAction = 'create' | 'read' | 'update' | 'replace' | 'delete' | 'noop' + +export interface DeployEvent extends BaseOutputMessage { + readonly type: 'deploy' + readonly status: DeployStatus + readonly action: DeployAction + readonly resource: string + + readonly state?: TfState['resources'][number] // Only relevant for `complete` statuses + + // waitingOn?: string[] // for pending states +} + +export interface FailedDeployEvent extends DeployEvent { + readonly type: 'deploy' + readonly status: 'failed' + readonly reason: string | Error +} + +// Emitted right before a deploy +export interface PlanEvent extends BaseOutputMessage { + readonly type: 'plan' + readonly plan: ParsedPlan +} + +// Emitted right after a deploy +export interface DeploySummaryEvent extends BaseOutputMessage { + readonly type: 'deploy-summary' + readonly add: number + readonly change: number + readonly remove: number + readonly errors?: Error[] +} + +// From custom resources +export interface DeployLogEvent extends BaseOutputMessage { + readonly type: 'deploy-log' + readonly level: 'trace' | 'debug' | 'info' | 'warn' | 'error' + readonly args: any[] + readonly resource: string +} + +// COMPILE + +// Emitted whenever a new template has been created +export interface CompileEvent extends BaseOutputMessage { + readonly type: 'compile' + readonly entrypoint: string + readonly template: any +} + + +// Using `console` or `process.(stdout|stderr)` during synthesis will fire this event +export interface SynthLogEvent extends BaseOutputMessage { + readonly type: 'synth-log' + readonly level: 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'raw' + readonly args: any[] +} + +// TESTS + +export interface BaseTestEvent extends BaseOutputMessage { + readonly type: 'test' + readonly id: number + readonly name: string + readonly parentId?: number + readonly context: OutputContext + readonly itemType: 'test' | 'suite' +} + +export interface PendingTestEvent extends BaseTestEvent { + readonly status: 'pending' +} + +export interface RunningTestEvent extends BaseTestEvent { + readonly status: 'running' +} + +export interface PassedTestEvent extends BaseTestEvent { + readonly status: 'passed' +} + +export interface FailedTestEvent extends BaseTestEvent { + readonly status: 'failed' + readonly reason: Error +} + +export type TestEvent = PendingTestEvent | RunningTestEvent | PassedTestEvent | FailedTestEvent + +export interface TestLogEvent extends BaseOutputMessage { + readonly type: 'test-log' + readonly level: 'trace' | 'debug' | 'info' | 'warn' | 'error' + readonly args: any[] + + // Test info + readonly id: number + readonly name: string + readonly parentId?: number +} + +type OutputEvent = LogEvent | PerfEvent | DeployEvent | TestEvent | CommandEvent | CompileEvent +type OutputEventTrigger = (message: Omit) => void + +const getSharedEmitter = memoize(createEventEmitter) + +export type Logger = ReturnType +function createLogger() { + const emitter = createEventEmitter() + const logEvent: OutputEventEmitter = createOutputEvent(emitter, 'log') + const perfEvent: OutputEventEmitter = createOutputEvent(emitter, 'perf') + const deployEvent: OutputEventEmitter = createOutputEvent(emitter, 'deploy') + const compileEvent: OutputEventEmitter = createOutputEvent(emitter, 'compile') + const testEvent: OutputEventEmitter = createOutputEvent(emitter, 'test') + const commandEvent: OutputEventEmitter = createOutputEvent(emitter, 'command') + + const testLogEvent: OutputEventEmitter = createOutputEvent(emitter, 'test-log') + const synthLogEvent: OutputEventEmitter = createOutputEvent(emitter, 'synth-log') + const deployLogEvent: OutputEventEmitter = createOutputEvent(emitter, 'deploy-log') + + const planEvent: OutputEventEmitter = createOutputEvent(emitter, 'plan') + const deploySummaryEvent: OutputEventEmitter = createOutputEvent(emitter, 'deploy-summary') + const resolveConfigEvent: OutputEventEmitter = createOutputEvent(emitter, 'resolve-config') + + const installEvent: OutputEventEmitter = createOutputEvent(emitter, 'install-lifecycle') + const packageProgressEvent: OutputEventEmitter = createOutputEvent(emitter, 'install-package') + + let perfCount = 0 + let perfObs: ReturnType | undefined + addMetaListener(emitter, ev => { + if (ev.eventName === 'perf') { + perfCount += ev.mode === 'added' ? 1 : -1 + } + + if (perfCount === 0) { + perfObs?.disconnect() + perfObs = undefined + } else if (!perfObs) { + perfObs = createObserver(perfEvent.fire) + } + }) + + return { + log: (...args: any[]) => logEvent.fire({ level: 'info', args }), + warn: (...args: any[]) => logEvent.fire({ level: 'warn', args }), + error: (...args: any[]) => logEvent.fire({ level: 'error', args }), + debug: (...args: any[]) => logEvent.fire({ level: 'debug', args }), + trace: (...args: any[]) => logEvent.fire({ level: 'trace', args }), + raw: (data: string | Uint8Array) => logEvent.fire({ level: 'raw', args: [data] }), + + emit: emitter.emit.bind(emitter), + emitDeployEvent: deployEvent.fire, + emitCompileEvent: compileEvent.fire, + emitTestEvent: testEvent.fire, + emitCommandEevent: commandEvent.fire, + emitTestLogEvent: testLogEvent.fire, + emitPlanEvent: planEvent.fire, + emitDeploySummaryEvent: deploySummaryEvent.fire, + emitResolveConfigEvent: resolveConfigEvent.fire, + emitSynthLogEvent: synthLogEvent.fire, + emitDeployLogEvent: deployLogEvent.fire, + emitInstallEvent: installEvent.fire, + emitPackageProgressEvent: packageProgressEvent.fire, + + onLog: logEvent.on, + onPerf: perfEvent.on, + onDeploy: deployEvent.on, + onCompile: compileEvent.on, + onTest: testEvent.on, + onTestLog: testLogEvent.on, + onCommand: commandEvent.on, + onPlan: planEvent.on, + onDeploySummary: deploySummaryEvent.on, + onResolveConfig: resolveConfigEvent.on, + onSynthLog: synthLogEvent.on, + onDeployLog: deployLogEvent.on, + onInstall: installEvent.on, + onPackageProgress: packageProgressEvent.on, + + dipose: () => { + emitter.removeAllListeners() + perfObs?.disconnect() + }, + } +} + +let logger: Logger +export function getLogger() { + return logger ??= createLogger() +} + +export function createConsole(logger: Logger): typeof globalThis.console { + return logger as any +} + +export function listenAll(logger: Logger, listener: (ev: OutputEvent) => void) { + const emitter = new EventEmitter() + emitter.on('any', listener) + + function emit(ev: OutputEvent) { + emitter.emit('any', ev) + } + + const disposeables = [ + logger.onLog(emit), + logger.onPerf(emit), + logger.onDeploy(emit), + logger.onCompile(emit), + logger.onTest(emit), + logger.onCommand(emit), + ] + + return { + dispose: () => { + emitter.removeAllListeners() + for (const d of disposeables) { + d.dispose() + } + }, + } +} + +type OutputEventEmitter = Omit, 'fire'> & { fire: OutputEventTrigger } + +function createOutputEvent(emitter: EventEmitter, type: U): OutputEventEmitter { + const event = createEvent(emitter, type) + + return { + on: event.on, + fire: message => { + const base: BaseOutputMessage = { + type, + timestamp: new Date(), + context: getOutputContext(), + } + + return event.fire(Object.assign(message, base)) + }, + } +} + +export function getTypedEventEmitter(type: U): OutputEventEmitter { + return createOutputEvent(getSharedEmitter(), type) +} \ No newline at end of file diff --git a/src/optimizer.ts b/src/optimizer.ts new file mode 100644 index 0000000..e1622f3 --- /dev/null +++ b/src/optimizer.ts @@ -0,0 +1,569 @@ +import ts from 'typescript' +import { type Symbol, createGraph, getContainingScope, liftScope } from './static-solver/scopes' +import { getNullTransformationContext, printNodes, toSnakeCase } from './utils' +import type { ExternalValue, ResourceValue, SerializedObject } from './runtime/modules/serdes' +import { isDataPointer } from './build-fs/pointers' +import { isModuleExport } from './permissions' + +function getCapturedNodes(sf: ts.SourceFile) { + const exportStatement = sf.statements.find(isModuleExport) + if (!exportStatement) { + return + } + + const closure = exportStatement.expression.right + if (!ts.isFunctionExpression(closure) || !ts.isBlock(closure.body)) { + return + } + + return closure.parameters +} + +function getClassDecl(sf: ts.SourceFile) { + const decl = sf.statements.find(ts.isFunctionDeclaration) ?? sf.statements.find(isModuleExport)?.expression.right + const body = (decl as any)?.body + if (!decl || !body || !ts.isBlock(body)) { + return + } + + return body.statements.find(ts.isClassDeclaration) +} + +// It's safe to do this when a complete closure graph contains class prototypes +// that are strictly referenced in terms of instances _and_ nothing attempts to +// call the class constructor indirectly via `.constructor` (very unlikely) +// +function pruneClassInitializers(sf: ts.SourceFile) { + // TODO: support ESM export declarations + const decl = sf.statements.find(ts.isFunctionDeclaration) ?? sf.statements.find(isModuleExport)?.expression.right + const body = (decl as any)?.body + if (!decl || !body || !ts.isBlock(body)) { + return + } + + + const closure = body.parent as ts.FunctionExpression + + // TODO: find class expression + const classDecl = body.statements.find(ts.isClassDeclaration) + if (!classDecl || !classDecl.name) { + return + } + + function visit(node: ts.Node): ts.Node { + if (ts.isClassDeclaration(node) && node.name && node.parent === body) { + const pruned = node.members.filter(x => { + if (ts.isConstructorDeclaration(x) || ts.isPropertyDeclaration(x)) { + return false + } + return true + }) + + return ts.factory.updateClassDeclaration( + node, + node.modifiers, + node.name, + node.typeParameters, + node.heritageClauses, + pruned + ) + } + + return ts.visitEachChild(node, visit, getNullTransformationContext()) + } + + const optimized = visit(sf) + + const graph = createGraph(optimized) + const sym = graph.symbols.get(classDecl.name) ?? graph.symbols.get(classDecl) + if (!sym || !sym.declaration) { + throw new Error(`Missing symbol for node: ${classDecl.name?.getText()}`) + } + const scope = getContainingScope(sym) + const res = liftScope(scope) + + const newCaptured = res.captured + const captured = closure.parameters + + function visit2(node: ts.Node): ts.Node { + if (ts.isFunctionExpression(node)) { + return ts.factory.updateFunctionExpression( + node, + node.modifiers, + node.asteriskToken, + node.name, + node.typeParameters, + newCaptured.map(sym => ts.factory.createParameterDeclaration(undefined, undefined, sym.name, undefined, undefined, undefined)), + node.type, + node.body, + ) + } + return ts.visitEachChild(node, visit2, getNullTransformationContext()) + } + + const updatedCaptured = captured.map(x => !!newCaptured.find(sym => sym.name === x.getText())) + + return { + pruned: visit2(optimized) as ts.SourceFile, + captured: updatedCaptured, + } +} + +function resolveNewExpressions(sf: ts.SourceFile) { + const resolved: ts.Node[] = [] + + function visit(node: ts.Node): void { + if (!ts.isNewExpression(node)) { + return ts.forEachChild(node, visit) + } + + resolved.push(node.expression) + + if (node.arguments) { + for (const arg of node.arguments) { + ts.forEachChild(arg, visit) + } + } + } + + visit(sf) + + return resolved +} + +const moveableStr = '@@__moveable__' + +// TODO: strip out logic added to classes to make them serializable if it's not needed + +// Must do two passes for removing class initializers: +// 1. Determine what things are being used as constructors (if any) + which things are classes +// 2. Prune all classes that are not used as constructors + +// This optimization looks everywhere an object might appear and sees which +// properties or methods are accessed. From there, we can build up a list of +// properties that are never used. +// +// We try to be conservative for computed element accesses. If we find such +// an expression and we cannot figure out all possible values then we assume +// any property could be accessed. +function pruneUnusedProperties( + table: Record, + captured: any, + getSourceFile: (pointer: string) => ts.SourceFile, +) { + function resolve(val: any) { + if (val && typeof val === 'object') { + if (moveableStr in val) { + const id = val[moveableStr].id + if (id !== undefined) { + return table[id] + } + } + } + return val + } + + const skipOptimize = new Set() + const usedProps = new Map>() + const methodCalls = new Map>() // FIXME: doesn't handle symbols + const symbols = new Map() + + for (const [k, val] of Object.entries(table)) { + if (!val.captured) continue + + const sf = getSourceFile(val.module) + const params = getCapturedNodes(sf) + if (!params) { + continue + } + + const graph = createGraph(sf) + const symbolMap = new Map() + for (let i = 0; i < params.length; i++) { + const sym = graph.symbols.get(params[i]) + if (!sym) continue + + symbolMap.set(sym, resolve(val.captured[i])) + } + + for (const [sym, v] of symbolMap) { + if (!v || (v.valueType !== 'object' && v.valueType !== 'resource')) continue + + const obj = v as SerializedObject | ResourceValue + const props = usedProps.get(obj) ?? new Set() + for (const r of sym.references) { + const exp = r.parent + if (ts.isPropertyAccessExpression(exp)) { + if (ts.isCallExpression(exp.parent)) { + if (exp.parent.expression === exp) { + if (obj.valueType === 'object' && obj.constructor) { + const calls = methodCalls.get(obj) ?? new Set() + calls.add(exp.name) + methodCalls.set(obj, calls) + } else { + // TODO + } + } + // TODO: maybe skip optimize here + + continue + } + + props.add(exp.name.text) + continue + } else if (ts.isElementAccessExpression(exp)) { + if (ts.isStringLiteral(exp.argumentExpression)) { + props.add(exp.argumentExpression.text) + } else { + skipOptimize.add(obj) + break + } + } else { + skipOptimize.add(obj) + break + } + } + + usedProps.set(obj, props) + } + } + + for (const [k, v] of methodCalls) { + const ctor = resolve(k.constructor!) + if (!ctor.module) { + skipOptimize.add(k) + continue + } + + const sf = getSourceFile(ctor.module) + // TODO: use the called method instead of looking at the whole class + const decl = getClassDecl(sf) + if (!decl) { + skipOptimize.add(k) + continue + } + + // TODO: deal with super classes + if (decl.heritageClauses) { + skipOptimize.add(k) + continue + } + + const graph = createGraph(sf) + const symbol = graph.symbols.get(decl) + const thisSymbol = symbol?.parentScope?.thisSymbol + if (!thisSymbol) { + skipOptimize.add(k) + continue + } + + symbols.set(k, thisSymbol) + + const props = usedProps.get(k) ?? new Set() + for (const ref of thisSymbol.references) { + const exp = ref.parent + if (ts.isPropertyAccessExpression(exp)) { + props.add(exp.name.getText()) + } else if (ts.isElementAccessExpression(exp)) { + if (ts.isStringLiteral(exp.argumentExpression)) { + props.add(exp.argumentExpression.text) + } else { + skipOptimize.add(k) + break + } + } else if (ts.isCallExpression(exp)) { + continue + } else { + skipOptimize.add(k) + break + } + } + + usedProps.set(k, props) + } + + function pruneResource(r: ResourceValue, props: Set): ExternalValue { + const capitalize = (s: string) => s ? s.charAt(0).toUpperCase().concat(s.slice(1)) : s + function normalize(str: string) { + const [first, ...rest] = str.split('_') + + return [first, ...rest.map(capitalize)].join('') + } + + const value = { ...r.value } + for (const k of Object.keys(value)) { + // Terraform uses snake_case for property names + if (!props.has(normalize(k))) { + delete value[k] + } + } + + return { ...r, value } as ExternalValue + } + + function pruneObject(obj: SerializedObject, props: Set): ExternalValue { + const properties = { ...obj.properties } + for (const k of Object.keys(properties)) { + if (!props.has(k)) { + delete properties[k] + } + } + + return { ...obj, properties } as any as ExternalValue + } + + const pruneableResources = new Map() + const newTable: Record = { ...table } + + function prune() { + for (const [k, v] of usedProps) { + if (skipOptimize.has(k)) continue + + if (k.valueType === 'object') { + const pruned = pruneObject(k, v) + for (const [k2, v2] of Object.entries((pruned as SerializedObject).properties ?? {})) { + if (typeof v2 === 'object' && moveableStr in v2) { + const id = v2[moveableStr].id + if (id !== undefined) { + const resolved = table[id] + if (resolved.valueType === 'resource') { + pruneableResources.set(k, [k2, resolved as ResourceValue]) + } + } + } + } + + newTable[k.id] = pruned + } else if (k.valueType === 'resource') { + newTable[k.id] = pruneResource(k, v) + } + } + } + + prune() + usedProps.clear() + + if (pruneableResources.size > 0) { + for (const [prop, [key, value]] of pruneableResources) { + const sym = symbols.get(prop)?.members.get(key) + if (!sym) continue + + // TODO: this is copy-pasted from above + const props = usedProps.get(value) ?? new Set() + for (const ref of sym.references) { + const exp = ref.parent + if (ts.isPropertyAccessExpression(exp)) { + props.add(exp.name.getText()) + } else if (ts.isElementAccessExpression(exp)) { + if (ts.isStringLiteral(exp.argumentExpression)) { + props.add(exp.argumentExpression.text) + } else { + skipOptimize.add(value) + break + } + } else if (ts.isCallExpression(exp)) { + continue + } else { + skipOptimize.add(value) + break + } + } + + if (props.size > 0) { + usedProps.set(value, props) + } + } + + prune() + } + + return { + table: newTable, + captured, + } +} + +export function optimizeSerializedData( + table: Record, + captured: any, + getSourceFile: (pointer: string) => ts.SourceFile, + writeDataSync: (buf: Uint8Array) => string +) { + const files = new Map() + const _getSourceFile = (pointer: string) => { + if (files.has(pointer)) { + return files.get(pointer)! + } + + const sf = getSourceFile(pointer) + files.set(pointer, sf) + + return sf + } + + const newExpressions = new Map() + const constructors = new Map() + function visitValue(id: string | number) { + const val = table[id] + if (val.valueType === 'object') { + const o = val as SerializedObject + if (o.constructor) { + const resolved = resolve(o.constructor) + if (typeof resolved !== 'function') { + constructors.set(id, resolved) + } + } else if (o.prototype) { + constructors.set(id, resolve(o.prototype)) + } + } + + if (val.valueType !== 'function') { + return + } + + const module = val.module + const sf = _getSourceFile(module) + const exps = resolveNewExpressions(sf) + if (exps.length > 0) { + newExpressions.set(id, exps) + } + } + + for (const id of Object.keys(table)) { + visitValue(id) + } + + function resolve(val: any) { + if (val && typeof val === 'object') { + if (moveableStr in val) { + const id = val[moveableStr].id + if (id !== undefined) { + return table[id] + } + } + } + return val + } + + // Now we need to evaluate the expressions to see if they match with any serialized values + const constructed = new Map>() + for (const [k, v] of newExpressions) { + const val = table[k] + if (!val.captured) continue + + const sf = v[0].getSourceFile() + const params = getCapturedNodes(sf) + if (!params) { + continue + } + + const graph = createGraph(sf) + const symbolMap = new Map() + for (let i = 0; i < params.length; i++) { + const sym = graph.symbols.get(params[i]) + if (!sym) continue + + symbolMap.set(sym, val.captured[i]) + } + + for (const n of v) { + // FIXME: we need to symbolically evaluate each expression + // This impl. does not handle dynamic construction + const sym = graph.symbols.get(n) + if (!sym) { + continue + } + + const val = symbolMap.get(sym) + const resolved = resolve(val) + if (resolved) { + const set = constructed.get(resolved) ?? new Set() + set.add(val) + constructed.set(resolved, set) + } + } + } + + const pruned = new Map() + function canPrune(val: ExternalValue) { + if (!maybeClasses.has(val)) { + return false + } + + const instantiators = constructed.get(val) + if (instantiators && [...instantiators].filter(x => !pruned.has(x)).length > 0) { + return false + } + + return true + } + + const maybeClasses = new Set(constructors.values()) + function maybePruneValue(id: string | number): ExternalValue { + const val = table[id] + if (!canPrune(val)) { + return visit(val) + } + + const sf = _getSourceFile(val.module) + const optimized = pruneClassInitializers(sf) + if (!optimized) { + return visit(val) + } + + const printed = printNodes(optimized.pruned.statements) + const newData = { + kind: 'compiled-chunk', + runtime: Buffer.from(printed).toString('base64'), + } + + const pointer = writeDataSync(Buffer.from(JSON.stringify(newData))) + const res: ExternalValue = { + ...val, + module: pointer, + captured: val.captured?.filter((_, i) => optimized.captured[i]), + } + + pruned.set(val, res) + + return visit(res) + } + + const newTable: Record = { ...table } + function visit(obj: any): any { + if (Array.isArray(obj)) { + return obj.map(visit) + } + + if (typeof obj !== 'object' || !obj) { + return obj + } + + // We won't touch these even though they might technically be a dynamically imported module + if (isDataPointer(obj)) { + return obj + } + + if (moveableStr in obj) { + const id = obj[moveableStr].id + if (id !== undefined) { + newTable[id] = maybePruneValue(id) + } + + return obj + } + + const r: Record = {} + for (const [k, v] of Object.entries(obj)) { + r[k] = visit(v) + } + return r + } + + const optimized = { + table: newTable, + captured: visit(captured), + } + + return pruneUnusedProperties(optimized.table, optimized.captured, _getSourceFile) +} \ No newline at end of file diff --git a/src/perf/profiles.ts b/src/perf/profiles.ts new file mode 100644 index 0000000..dae8390 --- /dev/null +++ b/src/perf/profiles.ts @@ -0,0 +1,300 @@ +import * as path from 'node:path' +import * as url from 'node:url' +import { Fs, SyncFs } from '../system' +import { SourceMapParser, createSourceMapParser, isSourceOrigin } from '../runtime/loader' + +interface CallFrame { + readonly url: string + readonly scriptId: string + readonly lineNumber: number + readonly columnNumber: number + readonly functionName: string +} + +interface CpuProfileNode { + readonly id: number + readonly callFrame: CallFrame + readonly hitCount: number + readonly children?: CpuProfileNode['id'][] + readonly positionTicks?: { + readonly line: number + readonly ticks: number + }[] +} + +interface CpuProfile { + readonly nodes: CpuProfileNode[] + readonly startTime: number + readonly endTime: number + readonly samples: number[] + readonly timeDeltas: number[] +} + +interface ProfileOptions { + readonly name?: string // filename + readonly interval?: number // in microseconds (default: 1000) +} + +export function getCommandLineOptions(opt?: ProfileOptions) { + const args = ['--cpu-prof'] + + if (opt?.name) { + args.push('--cpu-prof-name', opt.name) + } + + if (opt?.interval) { + args.push('--cpu-prof-interval', String(opt.interval)) + } + + return args +} + +interface ParsedCallFrame extends Omit { + readonly url?: URL + readonly originalFrame: CallFrame +} + +interface ParsedNode extends Omit { + readonly callFrame: ParsedCallFrame +} + +function applyRootMappings(fileName: string, rootMappings?: Record): string { + if (!rootMappings) { + return fileName + } + + for (const [k, v] of Object.entries(rootMappings)) { + if (fileName.startsWith(k)) { + return `${v}${fileName.slice(k.length)}` + } + } + + return fileName +} + +function getSourceMap(sourcemapParser: SourceMapParser, frame: CallFrame) { + if (!frame.url) { + return + } + + const frameUrl = new URL(frame.url) + if (frameUrl.protocol !== 'file:') { + return + } + + const fileName = url.fileURLToPath(frameUrl.href) + + return sourcemapParser.tryParseSourcemap(fileName, true) +} + +function parseCallFrame(sourcemapParser: SourceMapParser, frame: CallFrame, rootMappings?: Record): ParsedCallFrame { + if (!frame.url) { + return { ...frame, url: undefined, originalFrame: frame } + } + + const frameUrl = new URL(frame.url) + const parsed = { ...frame, url: frameUrl, originalFrame: frame } + if (frameUrl.protocol !== 'file:') { + return parsed + } + + // TODO: if `lineNumber` is 0 and `columnNumber` is 11 then it's _probably_ a file being loaded + // We should represent that somehow + + const fileName = url.fileURLToPath(frameUrl.href) + const sourcemap = sourcemapParser.tryParseSourcemap(fileName, true) + if (!sourcemap) { + return parsed + } + + // This is probably an import + if (frame.lineNumber === 0 && frame.columnNumber === 0 && !frame.functionName) { + // XXX: `findByEntry` doesn't work in this case so we have to infer + const sources = sourcemap.payload.sources + if (sources.length === 0 || sources.length > 1) { + return parsed + } + + const originalFile = applyRootMappings(path.resolve(path.dirname(fileName), sources[0]), rootMappings) + + return { + ...parsed, + url: url.pathToFileURL(originalFile), + functionName: '(import)', + } + } + + const mapping = sourcemap.findOrigin(frame.lineNumber + 1, frame.columnNumber + 1) + if (!isSourceOrigin(mapping)) { + return parsed + } + + const originalFile = applyRootMappings(path.resolve(path.dirname(fileName), mapping.fileName), rootMappings) + + return { + ...parsed, + url: url.pathToFileURL(originalFile), + lineNumber: mapping.lineNumber - 1, + columnNumber: mapping.columnNumber - 1, + functionName: mapping.name ?? frame.functionName, + } +} + +function parseNode(sourcemapParser: SourceMapParser, node: CpuProfileNode, rootMappings?: Record): ParsedNode { + const positionTicks = node.positionTicks ? (function () { + const sourcemap = getSourceMap(sourcemapParser, node.callFrame) + if (!sourcemap) { + return node.positionTicks! + } + + return node.positionTicks!.map(p => { + const mapping = sourcemap.findOrigin(p.line + 1, node.callFrame.columnNumber + 1) + if (!isSourceOrigin(mapping)) { + return p + } + + return { + line: mapping.lineNumber - 1, + ticks: p.ticks, + } + }) + })() : undefined + + return { + ...node, + positionTicks, + callFrame: parseCallFrame(sourcemapParser, node.callFrame, rootMappings), + } +} + +function printLink(frame: ParsedCallFrame, line?: number, workingDirectory?: string) { + if (!frame.url) { + return + } + + const lineNumber = (line ?? frame.lineNumber) + 1 + const columnNumber = line !== undefined ? 1 : frame.columnNumber + 1 + const fileName = frame.url.protocol === 'file:' ? url.fileURLToPath(frame.url) : frame.url.href + const rel = workingDirectory && fileName.startsWith(workingDirectory) ? path.relative(workingDirectory, fileName) : fileName + + return `${rel}:${lineNumber}:${columnNumber}` +} + +function printNode(node: ParsedNode, workingDirectory?: string) { + const link = printLink(node.callFrame, undefined, workingDirectory) + const name = node.callFrame.functionName + + return `${name ? name : '(anonymous)'}${link ? ` ${link}` : ''}` +} + +export async function loadCpuProfile(fs: Fs & SyncFs, fileName: string, workingDirectory: string, rootMappings?: Record) { + const sourcemapParser = createSourceMapParser(fs, undefined, workingDirectory) + const data: CpuProfile = JSON.parse(await fs.readFile(path.resolve(workingDirectory, fileName), 'utf-8')) + + const totalSamples = data.samples.reduce((a, b) => a + b, 0) + + const nodes = new Map(data.nodes.map(n => [n.id, parseNode(sourcemapParser, n, rootMappings)] as const)) + + const totalHits = new Map() + function getTotalHits(node: ParsedNode) { + if (totalHits.has(node.id)) { + return totalHits.get(node.id)! + } + + let t = node.hitCount + if (node.children) { + for (const id of node.children) { + t += getTotalHits(nodes.get(id)!) + } + } + + totalHits.set(node.id, t) + + return t + } + + const byFile = new Map() + for (const n of nodes.values()) { + if (n.callFrame.url?.protocol !== 'file:') { + continue + } + + const p = url.fileURLToPath(n.callFrame.url) + if (!byFile.has(p)) { + byFile.set(p, []) + } + + byFile.get(p)!.push(n) + } + + const filesOnly = true + const ignoreNodeModules = true + + // Groups all nodes by their callsite + const groupedNodes = new Map() + for (const n of nodes.values()) { + if (filesOnly && n.callFrame.url?.protocol !== 'file:') { + continue + } + if (ignoreNodeModules && n.callFrame.url?.pathname.includes('node_modules')) { + continue + } + + const l = printLink(n.callFrame) + if (!l) { + continue + } + + if (!groupedNodes.has(l)) { + groupedNodes.set(l, []) + } + + groupedNodes.get(l)!.push(n) + } + + const seen = new Set() + let hitsSum = 0 + + const r: [total: number, self: number, node: ParsedNode][] = [] + + for (const [k, v] of groupedNodes.entries()) { + const deduped = new Set() + function visit(n: ParsedNode) { + deduped.add(n.id) + if (n.children) { + for (const id of n.children) { + visit(nodes.get(id)!) + } + } + } + + v.forEach(visit) + + let t = 0 + for (const id of deduped) { + const hits = nodes.get(id)!.hitCount + t += hits + if (!seen.has(id)) { + hitsSum += hits + seen.add(id) + } + } + + const self = v.reduce((a, b) => a + b.hitCount, 0) + + r.push([t, self, v[0]]) + } + + const sortedByTotal = [...r].sort((a, b) => (b[0] - a[0])) + const sortedBySelf = [...r].sort((a, b) => (b[1] - a[1])) + + const toPercentage = (hits: number) => Math.floor((hits / hitsSum) * 10000) / 100 + + return [ + 'Sorted by total', + ...sortedByTotal.slice(0, 25).map(x => `${toPercentage(x[0])}% ${toPercentage(x[1])}% ${printNode(x[2], workingDirectory)}`), + '---------------------', + 'Sorted by self', + ...sortedBySelf.slice(0, 25).map(x => `${toPercentage(x[0])}% ${toPercentage(x[1])}% ${printNode(x[2], workingDirectory)}`) + ] +} diff --git a/src/permissions.ts b/src/permissions.ts new file mode 100644 index 0000000..f1812c7 --- /dev/null +++ b/src/permissions.ts @@ -0,0 +1,784 @@ +import ts from 'typescript' +import { AsyncLocalStorage, AsyncResource } from 'async_hooks' +import { Fn, isElement, isGeneratedClassConstructor, isOriginalProxy, isRegExp } from './runtime/modules/terraform' +import { createStaticSolver, createUnknown, getFunctionLength, isUnion, isUnknown, isInternalFunction, getSourceCode, evaluate as evaluateUnion } from './static-solver' +import { getLogger } from './logging' +import { dedupe } from './utils' +import { getArtifactOriginalLocation } from './runtime/loader' + +export function isModuleExport(node: ts.Node): node is ts.ExpressionStatement & { expression: ts.BinaryExpression } { + if (!ts.isExpressionStatement(node)) { + return false + } + + const exp = node.expression + if (!ts.isBinaryExpression(exp) || exp.operatorToken.kind !== ts.SyntaxKind.EqualsToken) { + return false + } + + return ( + ts.isPropertyAccessExpression(exp.left) && + exp.left.name.text === 'exports' && + ts.isIdentifier(exp.left.expression) && + exp.left.expression.text === 'module' + ) +} + +export function createSolver(substitute?: (node: ts.Node) => any) { + const MockNumber = new Proxy(Number, { + apply: (target, thisArg, args) => { + if (args.some(a => isUnknown(a) || isUnion(a))) { + return args[0] + // return new Proxy({}, { + // get: (_, p) => { + // if (p === Symbol.toPrimitive) { + // return (hint?: string) => { + // if (hint === 'number') { + // return NaN + // } else { + // return '*' + // } + // } + // } + // } + // }) + } + + return (target as any).apply(thisArg, args) + }, + }) + + class MockPromise { + static all(args: any[]) { + return args + } + } + + const MockObject = new Proxy({}, { + get: (target, prop, recv) => { + if (prop === 'assign') { + return function assign(target: any, ...sources: any[]) { + if (isUnknown(target)) { + return target + } + + if (target === undefined) { + return createUnknown() + } + + return Object.assign(target, ...sources) + } + } + + if (prop === 'entries') { + return function entries(target: any) { + if (isUnknown(target) || isUnion(target)) { + return [[createUnknown(), createUnknown()]] + } + + // XXX: not technically correct + if (!target) { + return [[createUnknown(), createUnknown()]] + } + + return Object.entries(target) + } + } + + if (prop === 'values') { + return function values(target: any) { + if (isUnknown(target) || isUnion(target)) { + return Object.assign([createUnknown()], { + splice: (start: number, count: number) => { + return [createUnknown()] + }, + }) + } + + if (!target) { + return createUnknown() + } + + // XXX: using the derived value is not always correct + // The only time we can return an empty array is if we can guarantee that nothing + // would be assigned to the target object + const values = Object.values(target) + if (values.length === 0) { + return Object.assign([createUnknown()], { + splice: (start: number, count: number) => { + return [createUnknown()] + }, + }) + } + + return values + } + } + + if (prop === 'keys') { + return function keys(target: any) { + if (!target) { + return createUnknown() + } + if (isUnknown(target)) { + return [target] + } + + return Object.keys(target) + } + } + + if (prop === 'getPrototypeOf') { + return function getPrototypeOf(target: any) { + if (!target) { + return createUnknown() + } + if (isUnknown(target)) { + return target + } + + return Object.getPrototypeOf(target) + } + } + + // if (prop === 'create') { + // return function create(val: any, desc?: Record) { + // if (isUnknown(val) || isUnknown(desc)) { + // return createUnknown() + // } + + // return desc ? Object.create(val, desc) : Object.create(val) + // } + // } + + return createUnknown() + } + }) + + + const solver = createStaticSolver((node, scope) => { + if (ts.isIdentifier(node) && node.text === 'require') { + return (id: string) => { + return createUnknown() + } + } + + if (ts.isIdentifier(node)) { + switch (node.text) { + case 'Array': + return Array + case 'Number': + return MockNumber + case 'Promise': + return MockPromise + case 'Object': + return MockObject + // case 'Symbol': + // return Symbol + case 'console': + return new Proxy(console, { + get: (target, prop, recv) => { + if (prop === 'log' || prop === 'warn' || prop === 'error') { + return (...args: any[]) => { + getLogger().log('permissions logger:', ...args.map(a => { + if (isInternalFunction(a)) { + return getSourceCode(a) + ' [wrapped]' + } + + return a + })) + } + } + + return target[prop as keyof typeof console] + }, + }) + case 'JSON': + return { + parse: (o: any) => { + return createUnknown() + }, + stringify: (val: any, replacer?: any, space?: any) => { // INCORRECT + if (isUnion(val)) { + evaluateUnion(val) + } + + return createUnknown() + }, + } + } + } + + return substitute?.(node) + }) + + return solver +} + +// Example Azure role for reading data from the Storage service: +// { +// "assignableScopes": [ +// "/" +// ], +// "description": "Allows for read access to Azure File Share over SMB", +// "id": "/subscriptions/{subscriptionId}/providers/Microsoft.Authorization/roleDefinitions/aba4ae5f-2193-4029-9191-0cb91df5e314", +// "name": "aba4ae5f-2193-4029-9191-0cb91df5e314", +// "permissions": [ +// { +// "actions": [], +// "notActions": [], +// "dataActions": [ +// "Microsoft.Storage/storageAccounts/fileServices/fileshares/files/read" +// ], +// "notDataActions": [] +// } +// ], +// "roleName": "Storage File Data SMB Share Reader", +// "roleType": "BuiltInRole", +// "type": "Microsoft.Authorization/roleDefinitions" +// } + +const original = Symbol.for('original') +const unproxy = Symbol.for('unproxy') +const moveable = Symbol.for('__moveable__') +const moveable2 = Symbol.for('__moveable__2') +const permissions = Symbol.for('permissions') +const unproxyParent = Symbol.for('unproxyParent') +const expressionSym = Symbol.for('expression') + +function isProxy(o: any, checkPrototype = false) { + return !!o && ((checkPrototype && unproxy in o) || !!Reflect.getOwnPropertyDescriptor(o, moveable2)) +} + +function substitute(template: any, args: any[], addStatement: (s: any) => void, thisArg?: any, context?: any): any { + if (typeof template === 'string') { + let result = template + + const matched: boolean[] = [] + for (let i = 0; i < args.length; i++) { + const arg = args[i] + const regexp = new RegExp(`\\{${i}\\.([A-Za-z0-9]+)\\}`, 'g') + result = result.replace(regexp, (_, prop) => { + matched[i] = true + // TODO: unions + if (isUnknown(arg[prop]) || isUnion(arg[prop])) { + return '*' + } + return arg[prop] ?? '*' + }) + } + + // ARGS2 + for (let i = 0; i < args.length; i++) { + if (matched[i]) continue + const arg = args[i] + const regexp = new RegExp(`\\{${i}}`, 'g') + result = result.replace(regexp, (_) => { + matched[i] = true + if (isUnknown(arg)) { + return '*' + } + return arg ?? '*' + }) + } + + // ARGS3 + // BIG HACK + for (let i = 0; i < args.length; i++) { + if (matched[i]) continue + const arg = args[i] + const regexp = new RegExp(`[^$]\\{([^\{\}]*)${i}\\.([A-Za-z0-9]+)([^\{\}]*)\\}`, 'g') + result = result.replace(regexp, (_, $1, $2, $3) => { + matched[i] = true + if (isUnknown(arg) || isUnknown(arg[$2])) { + return '*' + } + return `\{${$1}${arg[$2].toString().replace(/^\{/, '').replace(/\}$/, '')}${$3}\}` + }) + } + + if (context) { + if (original in context) { + context = context[original] + } + + // FIXME: `replace` thinks these are functions and tries to call them + result = result.replace(/\{context\.Partition\}/g, context.partition.toString()) + result = result.replace(/\{context\.Region\}/g, context.regionId.toString()) + result = result.replace(/\{context\.Account\}/g, context.accountId.toString()) + } + + return result + } else if (Array.isArray(template)) { + // XXX: ugly hack, need to clean-up the permissions API + if (typeof template[0] === 'string') { + return template.map(v => substitute(v, args, addStatement, thisArg, context)) + } + + template.forEach(v => substitute(v, args, addStatement, thisArg, context)) + + return createUnknown() + } else if (typeof template === 'function') { + if (context && original in context) { + context = context[original] + } + + const $context = Object.create(context ?? null, { + addStatement: { value: addStatement }, + createUnknown: { value: createUnknown }, + }) + + const thisArgWithCtx = Object.create(thisArg ?? null, { + $context: { value: $context } + }) + + return template.apply(thisArgWithCtx, args) + } else if (typeof template === 'object' && !!template) { + const result: any = {} + for (const [k, v] of Object.entries(template)) { + result[k] = substitute(v, args, addStatement, thisArg, context) + } + + addStatement(result) + + return createUnknown() + } else { + return String(template) + } +} + +function getModel(o: any): PermissionsBinding | undefined { + if (!o) return + + if (Object.prototype.hasOwnProperty.call(o, permissions)) { + const model = o[permissions] + if (model.type === 'class' || model.type === 'object' || model.type === 'function') { + return model + } + } +} + +export function createPermissionsBinder() { + // Does two things: + // 1. Loads in any models discovered + // 2. Infers the "permissions signature" of statements + + + + function createCapturedSolver( + getSourceFile: (fileName: string) => ts.SourceFile + ) { + const resolveCache = new Map() + const ctx = new AsyncLocalStorage<{ statements: any[]; canCache?: boolean }>() + + // Can probably cache most things now + function skipCache() { + const store = ctx.getStore() + if (store !== undefined) { + store.canCache = false + } + } + + function getStatements() { + return ctx.getStore()?.statements ?? [] + } + + function addStatement(s: any) { + s.Effect ??= 'Allow' + getStatements().push(s) + } + + function evaluate(target: any, getContext: (target: any) => any, globals?: { console?: any }, args: any[] = [], thisArg?: any) { + const solver = createSolver() + const factoryFunctions = new Map any>() + function getFactoryFunction(fileName: string) { + if (factoryFunctions.has(fileName)) { + return factoryFunctions.get(fileName)! + } + + const sf = getSourceFile(fileName) + const decl = sf.statements.find(ts.isFunctionDeclaration) ?? sf.statements.find(isModuleExport)?.expression.right + if (!decl) { + throw new Error(`No export found: ${fileName}`) + } + + const fn = solver.createSolver().solve(decl) + const factory = (...args: any[]) => fn.call([decl], undefined, ...args) + factoryFunctions.set(fileName, factory) + + return factory + } + + function createFunction(model: FunctionPermissionsBinding['call'], t: any) { + skipCache() + + return function (...args: any[]) { + return substitute(model, args, addStatement, t, getContext(t)) + } + } + + function createInstance(model: ObjectPermissionsBinding['methods'] | ClassPermissionsBinding['methods'], t: any) { + skipCache() + + const proto = {} as Record + for (const [k, v] of Object.entries(model)) { + proto[k] = function (...args: any[]) { + return substitute(v, args, addStatement, t, getContext(t)) + } + } + + return proto + } + + function createTree(model: any, t: any) { + if (model.type === 'function') { + return createFunction(model.call, t) + } + + if (model.type !== 'container') { + return function () { return createInstance(model.methods, t) } + } + + const res: Record = {} + for (const [k, v] of Object.entries(model.properties)) { + if (typeof v !== 'object' || v === null) { + throw new Error(`Unexpected permissions binding: ${v}`) + } + + if ((v as any).type) { + res[k] = createTree(v, t) + } + } + + return Object.assign(resolveObject(t), res) + } + + function createStubFromModel(model: PermissionsBinding, obj: any) { + if (model.type === 'class') { + return function (...args: any[]) { + if (model.$constructor) { + const t = thisArg ?? obj + substitute(model.$constructor, args, addStatement, t, getContext(t)) + } + + return createInstance(model.methods, obj) + } + } else if (model.type === 'object') { + return createInstance(model.methods, obj) + } else if (model.type === 'function') { + return createFunction(model.call, obj) + } + } + + function getBaseObject(o: any) { + if (!o.constructor || Object.getPrototypeOf(o) === null) { + return Object.create(null) + } else if (o.constructor.name === 'Object') { + return {} + } + + const ctor = resolve(o.constructor) + if (ctor && !isUnknown(ctor)) { + return Object.create(ctor.prototype, { + constructor: { + value: ctor, + enumerable: true, + configurable: true, + } + }) + } + + return createUnknown() + } + + function resolveObject(o: any) { + const resolved = getBaseObject(o) + resolveCache.set(o, resolved) // This cache is probably redundant with the heavy caching in `resolveProperties` + + if (isUnknown(resolved)) { + return resolved + } + + const _ctx = { canCache: true, statements: getStatements() } + ctx.run(_ctx, resolveProperties) + + if (!_ctx.canCache) { + skipCache() + resolveCache.delete(o) + } + + return resolved + + function resolveProperties() { + // Lazily-evaluate every enumerable property + for (const k of Object.keys(o)) { + let currentVal: any + let didResolve = false + Object.defineProperty(resolved, k, { + get: () => { + if (didResolve) { + return currentVal + } + + const _ctx = { canCache: true, statements: getStatements() } + const val = ctx.run(_ctx, () => resolve(o[k])) + + if (_ctx.canCache) { + didResolve = true + currentVal = val + } + + return val + }, + set: (nv) => { + didResolve = true + currentVal = nv + }, + configurable: true, + enumerable: true, + }) + } + } + } + + function resolve(o: any): any { + if ((typeof o !== 'object' && typeof o !== 'function') || !o) { + return o + } + + if (resolveCache.has(o)) { + return resolveCache.get(o) + } + + const _ctx = { canCache: true, statements: getStatements() } + const res = ctx.run(_ctx, inner) + + if (_ctx.canCache) { + resolveCache.set(o, res) + } else { + skipCache() + } + + return res + + function inner() { + if (expressionSym in o) { + if (!isOriginalProxy(o) || !o.constructor || isGeneratedClassConstructor(o.constructor)) { + return o + } + } + + if (Array.isArray(o)) { + return o.map(resolve) + } + + if (isProxy(o)) { + const unproxied = o[unproxy] + const ctor = unproxied.constructor + if (ctor) { + const ctorM = getModel(ctor[unproxy]) + if (ctorM && ctorM.type === 'class') { + return createInstance(ctorM.methods, unproxied) + } + } + + const p = o[unproxyParent] + const pm = getModel(p) + if (pm) { + return createStubFromModel(pm, unproxied) + } + + if (ctor && typeof ctor[moveable] === 'function') { + const resolved = resolveObject(unproxied) + const model = getModel(ctor) + + if (model && model.type !== 'function') { + // Merges the model into the current object + const partial = createInstance(model.methods, unproxied) + + return Object.assign(resolved, partial) + } + + if (isOriginalProxy(unproxied)) { + return new Proxy(resolved, { + get: (target, prop, recv) => { + if (Reflect.has(target, prop)) { + return target[prop] + } + + return unproxied[prop] + } + }) + } + + return resolved + } + + const opm = getModel(unproxied) + if (opm !== undefined) { + return createStubFromModel(opm, unproxied) + } + + if (typeof o[moveable] === 'function') { + const val = o[moveable]() + if (val.valueType === 'function') { + return invokeCaptured(val.module, resolve(substituteGlobals(val.captured))) + } else if (val.valueType === 'reflection') { + return createUnknown() + } + } + + const x = o[moveable2]() + if (x?.valueType === 'reflection') { + return createUnknown() + } + + return resolve(unproxied) + } + + const m = getModel(o) + if (m) { + return createStubFromModel(m, o) + } + + const ctor = o.constructor + const ctorModel = ctor ? getModel(ctor) : undefined + if (ctorModel && ctorModel.type !== 'function') { + const partial = createInstance(ctorModel.methods, o) + return Object.assign(resolveObject(o), partial) + } + + if (isUnion(o)) { + return o + } + + const perms = o[permissions] + if (perms) { + if (perms.type === 'container') { + return createTree(perms, o) + } + + return createStubFromModel(perms, o) + } + + if (typeof o[moveable] === 'function') { + const val = o[moveable]() + if (val.valueType === 'function') { + return invokeCaptured(val.module, resolve(substituteGlobals(val.captured))) + } else if (val.valueType === 'reflection') { + return createUnknown() + } + } + + if (isRegExp(o)) { + return o + } + + if (typeof o === 'object') { + return resolveObject(o) + } + + return createUnknown() + } + } + + function substituteGlobals(captured: any[]) { + if (!globals) { + return captured + } + + return captured.map(c => { + if ((typeof c === 'object' || typeof c === 'function') && !!c && typeof c[moveable2] === 'function') { + const desc = c[moveable2]() + if (desc.operations && desc.operations[0].type === 'global') { + const target = desc.operations[1] + if (target && target.type === 'get') { + return globals[target.property as keyof typeof globals] ?? c + } + } + } + + return c + }) + } + + function invokeCaptured(fileName: string, captured: any[]): any { + skipCache() // We'll keep a separate cache for re-hydrated factory functions + + return getFactoryFunction(fileName)(...captured) + } + + const statements: any[] = [] + try { + ctx.run({ statements }, () => { + const fn = resolve(target) + call(fn, args) + }) + } catch (e) { + const module = (typeof target === 'object' || typeof target === 'function') + ? target?.[moveable]?.().module + : undefined + + if (module) { + const location = getArtifactOriginalLocation(module) + if (location) { + throw new Error(`Failed to solve permissions for target: ${location}`, { cause: e }) + } + } + + throw new Error(`Failed to solve permissions for target: ${require('node:util').inspect(target)}`, { cause: e }) + } + + return dedupe(statements.flat(100)) + } + + return { evaluate } + } + + function call(fn: any, args: any[]) { + if (isUnknown(fn)) { + return + } + + if (isUnion(fn)) { + for (const f of fn) { + call(f, args) + } + + return + } + + if (isInternalFunction(fn)) { + fn.call([], undefined, ...args) + } else { + fn.call(undefined, ...args) + } + } + + return { createCapturedSolver } +} + +// STUB +type Model = any + +// TODO: come up with better name +// The logic is being used for permissions but is perfectly usuable for any kind of binding + + +interface ObjectPermissionsBinding { + type: 'object' + methods: Record +} + +interface ClassPermissionsBinding { + type: 'class' + methods: Record + $constructor?: Model +} + +interface FunctionPermissionsBinding { + type: 'function' + call: Model +} + +type PermissionsBinding = ObjectPermissionsBinding | ClassPermissionsBinding | FunctionPermissionsBinding \ No newline at end of file diff --git a/src/pm/attestations.ts b/src/pm/attestations.ts new file mode 100644 index 0000000..2e08808 --- /dev/null +++ b/src/pm/attestations.ts @@ -0,0 +1,50 @@ +// Variant of https://github.com/secure-systems-lab/dsse using UTF-8 for the payload + +export interface Envelope { + readonly payload: string // utf-8 + readonly payloadType: string + readonly signatures: { + readonly keyid: string + readonly sig: string // base64url + }[] +} + +export interface KeyPair { + readonly id: string + sign(data: Buffer): Promise + verify(data: Buffer, sig: Buffer): Promise +} + +function createHeader(envelope: Omit) { + const data = Buffer.from(envelope.payload, 'utf-8') + const type = Buffer.from(envelope.payloadType, 'utf-8') + + return Buffer.concat([ + Buffer.from(String(type.byteLength) + ' ', 'utf-8'), + type, + Buffer.from(' ' + String(data.byteLength) + ' ', 'utf-8'), + data, + ]) +} + +export async function sign(envelope: Omit, key: Pick) { + const header = createHeader(envelope) + const sig = Buffer.from(await key.sign(header)).toString('base64url') + + return { keyid: key.id, sig } +} + +export async function verify(envelope: Envelope, key: Pick) { + const signature = envelope.signatures[0] + if (!signature) { + throw new Error(`Envelope is missing a signature`) + } + + if (signature.keyid !== key.id) { + throw new Error(`Found different key ids: ${signature.keyid} !== ${key.id}`) + } + + const header = createHeader(envelope) + + return key.verify(header, Buffer.from(signature.sig, 'base64url')) +} \ No newline at end of file diff --git a/src/pm/compat.ts b/src/pm/compat.ts new file mode 100644 index 0000000..099b988 --- /dev/null +++ b/src/pm/compat.ts @@ -0,0 +1,127 @@ + +import * as path from 'node:path' +import { homedir } from 'node:os' +import { getFs } from '../execution' +import { isNonNullable, throwIfNotFileNotFoundError } from '../utils' + +// per-project config file (/path/to/my/project/.npmrc) +// per-user config file (~/.npmrc) +// global config file ($PREFIX/etc/npmrc) +// npm builtin config file (/path/to/npm/npmrc) + +// Array values are specified by adding "[]" after the key name. For example: foo[] = bar; foo[] = baz + +// _auth (base64 authentication string) +// _authToken (authentication token) +// username +// _password +// email +// certfile (path to certificate file) +// keyfile (path to key file) + +// ; good config +// @myorg:registry=https://somewhere-else.com/myorg +// @another:registry=https://somewhere-else.com/another +// //registry.npmjs.org/:_authToken=MYTOKEN +// ; would apply to both @myorg and @another +// ; //somewhere-else.com/:_authToken=MYTOKEN +// ; would apply only to @myorg +// //somewhere-else.com/myorg/:_authToken=MYTOKEN1 +// ; would apply only to @another +// //somewhere-else.com/another/:_authToken=MYTOKEN2 + +export interface NpmConfig { + readonly registries: Record + readonly scopedConfig: Record +} + +export function parseText(text: string) { + const lines = text.split('\n').map(x => x.trim()).filter(x => !!x) + const registries: Record = {} + const scopedConfig: Record> = {} + for (const l of lines) { + if (l.startsWith('#') || l.startsWith(';')) continue + + const m = l.match(/^([^\s=]+)\s*=\s*(.*?)\s*$/) + if (!m) { + throw new Error(`Bad parse: ${l}`) // FIXME: may contain sensitive stuff + } + + const [_, key, value] = m + + // Technically environemnt variables can be anything but a few special symbols e.g. "$\= + const resolvedValue = value.replace(/\$\{([A-Za-z0-9\-_]+)\}/g, (_, name) => { + const val = process.env[name] ?? process.env[`NPM_CONFIG_${name}`] + if (val === undefined) { + throw new Error(`No environment variable found: ${name}`) + } + return val + }) + + const scopedMatch = key.match(/^(@[a-z]+):registry$/) + if (scopedMatch) { + registries[scopedMatch[1]] = resolvedValue + continue + } + + const uriFragmentMatch = key.match(/^\/\/([^:]+):([a-zA-Z_]+)/) + if (uriFragmentMatch) { + const [_, scope, key] = uriFragmentMatch + const config = scopedConfig[scope] ??= {} + config[key] = resolvedValue + continue + } + } + + return { + registries, + scopedConfig, + } +} + +function mergeConfigs(configs: NpmConfig[]): NpmConfig { + if (configs.length === 1) { + return configs[0] + } + + const merged: NpmConfig = { registries: {}, scopedConfig: {} } + for (const c of configs) { + for (const [k, v] of Object.entries(c.registries)) { + if (!merged.registries[k]) { + merged.registries[k] = v + } + } + + for (const [k, v] of Object.entries(c.scopedConfig)) { + const scoped = merged.scopedConfig[k] ??= {} + for (const [k2, v2] of Object.entries(v)) { + if (!scoped[k2]) { + scoped[k2] = v2 + } + } + } + } + + return merged +} + +export async function resolveNpmConfigs(pkgDir: string) { + const sources = [ + path.resolve(homedir(), '.npmrc'), + path.resolve(pkgDir, '.npmrc'), + ] + + const texts = await Promise.all( + sources.map(p => getFs().readFile(p, 'utf-8').catch(throwIfNotFileNotFoundError)) + ) + + const configs = texts.map(text => text ? parseText(text) : undefined).filter(isNonNullable) + if (configs.length === 0) { + return + } + + return mergeConfigs(configs) +} diff --git a/src/pm/integrity.ts b/src/pm/integrity.ts new file mode 100644 index 0000000..1ca051b --- /dev/null +++ b/src/pm/integrity.ts @@ -0,0 +1,111 @@ +import path from 'node:path' +import * as fs from 'node:fs/promises' +import * as nodeStream from 'node:stream' +import { createHash, timingSafeEqual } from 'node:crypto' + +type HashSums = [string, Buffer][] + +function parseHashSums(text: string, hashEncoding: BufferEncoding = 'hex'): [string, Buffer][] { + const result: [string, Buffer][] = [] + for (const line of text.split('\n')) { + const match = line.match(/^([^\s]+) (.+)$/) + if (!match) { + throw new Error(`Failed to parse hash entry: ${line}`) + } + + const [_, hash, fileName] = match + const buf = Buffer.from(hash, hashEncoding) + result.push([fileName, buf]) + } + + return result +} + +function resolveHashSums(sums: HashSums, dir: string): HashSums { + return sums.map(([f, h]) => [path.relative(dir, f), h]) +} + +class IntegrityException extends Error { + public constructor( + public readonly filePath: string, + public readonly actual: Buffer, + public readonly expected: Buffer + ) { + super(`File ${filePath} failed integrity check`) + } +} + +async function checksum(filePath: string, expected: Buffer) { + const actual = await hashFile(filePath) + if (actual.byteLength !== expected.byteLength) { + return new IntegrityException(filePath, actual, expected) + } + + if (!timingSafeEqual(actual, expected)) { + return new IntegrityException(filePath, actual, expected) + } +} + +async function validateHashSums(sums: HashSums): Promise { + const result: IntegrityException[] = [] + for (const [f, h] of sums) { + const err = await checksum(f, h) + if (err) { + result.push(err) + } + } + return result +} + +async function hashStream(stream: ReadableStream, alg: 'sha256' = 'sha256') { + const h = createHash(alg) + const d = nodeStream.Transform.toWeb(h) + await stream.pipeTo(d.writable) + + return h.digest() +} + +async function hashFile(filePath: string, alg: 'sha256' = 'sha256') { + const r = await fs.open(filePath, 'r').then(handle => handle.createReadStream()) + const h = await hashStream(nodeStream.Readable.toWeb(r) as any, alg) + + return h +} + +export async function hashFiles(files: string[], workingDir: string, alg: 'sha256' = 'sha256') { + const result: [string, Buffer][] = [] + for (const f of files) { + const absPath = path.resolve(workingDir, f) + result.push([absPath, await hashFile(absPath, alg)]) + } + + return result +} + +export async function createShaSumsFileText(files: string[], workingDir: string) { + const sums = await hashFiles(files, workingDir) + const result: string[] = [] + for (const [f, h] of sums) { + const relPath = path.resolve(workingDir, f) + const encoded = h.toString('hex') + result.push(`${encoded} ${relPath}`) + } + + result.push('') + + return result.join('\n') +} + +export async function checkShaSumsFile(filePath: string, workingDir = path.dirname(filePath), include?: string[]) { + const text = await fs.readFile(filePath, 'utf-8') + const res = parseHashSums(text) + + const includeSet = include ? new Set(include.map(f => path.relative(workingDir, f))) : undefined + const filtered = includeSet ? res.filter(([f]) => includeSet.has(f)) : res + const resolved = resolveHashSums(filtered, workingDir) + + const errors = await validateHashSums(resolved) + if (errors.length > 0) { + throw new AggregateError(errors) + } +} \ No newline at end of file diff --git a/src/pm/manifests.ts b/src/pm/manifests.ts new file mode 100644 index 0000000..6b2e0c1 --- /dev/null +++ b/src/pm/manifests.ts @@ -0,0 +1,531 @@ +import * as path from 'node:path' +import { PackageJson } from './packageJson' +import { keyedMemoize, memoize } from '../utils' +import { getGlobalCacheDirectory } from '../workspaces' +import { createMemento } from '../utils/memento' +import { getFs } from '../execution' +import { createRequester } from '../utils/http' +import { getLogger } from '..' +import { NpmConfig, resolveNpmConfigs } from './compat' + +type NpmKeyId = `SHA256:${string}` +export interface PublishedPackageJson extends PackageJson { + dist: { + integrity: string + fileCount?: number + unpackedSize?: number // bytes + tarball: string // URL + // `sig` is generated by signing `${package.name}@${package.version}:${package.dist.integrity}` + signatures?: { keyid: NpmKeyId; sig: string }[] + + // Synapse only + isStubPackage?: boolean + isSynapsePackage?: boolean + } +} + +export interface PackageManifest { + name: string + versions: Record + 'dist-tags'?: Record +} + +interface NpmKeysResponse { + readonly keys: { + expires: null | string // ISO 8601 + keyid: NpmKeyId + keytype: string // ecdsa-sha2-nistp256 + scheme: string // ecdsa-sha2-nistp256 + key: string // base64 encoded public key + }[] +} + +export interface OptimizedPackageManifest { + readonly symbolTable: string[] + readonly tags?: Record + readonly versions: Record +} + +interface OptimizedPublishedPackageJson { + readonly dist: PublishedPackageJson['dist'] + readonly os?: PublishedPackageJson['os'] + readonly cpu?: PublishedPackageJson['cpu'] + readonly dependencies?: PublishedPackageJson['dependencies'] + readonly peerDependencies?: PublishedPackageJson['peerDependencies'] + readonly optionalDependencies?: PublishedPackageJson['optionalDependencies'] + readonly peerDependenciesMeta?: PublishedPackageJson['peerDependenciesMeta'] +} + + +function createKeyDeduper(symbols = new Map()) { + function getId(k: string) { + const id = symbols.get(k) + if (id !== undefined) { + return id + } + + const newId = `${symbols.size}` + symbols.set(k, newId) + + return newId + } + + function dedupe(obj: any): any { + if (Array.isArray(obj)) { + return obj.map(dedupe) + } + + if (typeof obj !== 'object' || !obj) { + return obj + } + + const res: any = {} + for (const [k, v] of Object.entries(obj)) { + res[getId(k)] = dedupe(v) + } + + return res + } + + function getSymbolTable() { + return [...symbols.entries()].sort((a, b) => Number(a[1]) - Number(b[1])).map(a => a[0]) + } + + return { dedupe, redupe: decodedWithTable, getSymbolTable } +} + +function decodedWithTable(obj: any, symbolTable: string[]) { + function getKey(id: string) { + const n = Number(id) + if (isNaN(n)) { + throw new Error(`Not an entry: ${id}`) + } + + const k = symbolTable[n] + if (k === undefined) { + throw new Error(`Missing key at index: ${n}`) + } + + return k + } + + function decode(obj: any): any { + if (Array.isArray(obj)) { + return obj.map(decode) + } + + if (typeof obj !== 'object' || !obj) { + return obj + } + + const res: any = {} + for (const [k, v] of Object.entries(obj)) { + res[getKey(k)] = decode(v) + } + + return res + } + + return decode(obj) +} + +// De-dupes and strips out unneeded things +function optimizePackageManfiest(manifest: PackageManifest): OptimizedPackageManifest { + const deduper = createKeyDeduper() + const versions: Record = {} + for (const [k, v] of Object.entries(manifest.versions)) { + const opt = optimizePublishedPackageJson(v) + versions[k] = deduper.dedupe(opt) + } + + return { + versions, + tags: manifest['dist-tags'], + symbolTable: deduper.getSymbolTable(), + } +} + +function hydratePackageJson(name: string, symbolTable: string[], pkgJson: OptimizedPublishedPackageJson) { + const c = decodedWithTable(pkgJson, symbolTable) + c.name = name + + return c +} + +// TODO: use wyhash for this +// function createRecordHasher() { +// const hashes = new Map() + +// function getStringHash(s: string) { +// const h = hashes.get(s) +// if (h) { +// return h +// } + +// const h2 = createHash('sha256').update(s).digest() +// hashes.set(s, h2) + +// return h2 +// } + +// function combineHash(a: Buffer, b: Buffer) { +// const c = Buffer.allocUnsafe(a.byteLength) +// for (let i = 0; i < a.byteLength; i += 4) { +// c.writeUint32LE((a.readUint32LE(i) * 3) + b.readUInt32LE(i), i) +// } +// return c +// } + +// function getHash(obj: Record) { +// // const entries = Object.entries(obj).sort((a, b) => a[0].localeCompare(b[0])) +// // We don't sort because we will XOR the KV pairs +// const entries = Object.entries(obj) +// const z = entries.map(([k, v]) => combineHash(getStringHash(k), getStringHash(v))) +// const c = z[0] +// for (let j = 0; j < c.byteLength; j++) { +// let x = c.readUint32LE(j) +// for (let i = 1; i < z.length; i++) { +// x ^= z[i].readUint32LE(j) +// } +// c.writeUint32LE(x, j) +// } + +// return c +// } + +// return { getHash } +// } + +function optimizePublishedPackageJson(pkgJson: PublishedPackageJson): OptimizedPublishedPackageJson { + return { + dist: pkgJson.dist, + os: pkgJson.os, + cpu: pkgJson.cpu, + dependencies: pkgJson.dependencies, + peerDependencies: pkgJson.peerDependencies, + optionalDependencies: pkgJson.optionalDependencies, + peerDependenciesMeta: pkgJson.peerDependenciesMeta, + } +} + +function createManifestCache(registry: string) { + const dir = path.resolve(getGlobalCacheDirectory(), 'package-manifests', registry) + const memento = createMemento(getFs(), dir) + + return memento +} + +const getManifestCache = keyedMemoize(createManifestCache) + +export function createManifestRepo(client = createNpmRegistryClient(), manifestCache = getManifestCache('npm')) { + const manifestAbortController = new AbortController() + + const events = require('node:events') as typeof import('node:events') + events.setMaxListeners(50, manifestAbortController.signal) + + const cancelError = new Error('Cancelled') + function cancelManifestRequests() { + manifestAbortController.abort(cancelError) + } + + // Decent amount of time is still spent parsing JSON + const manifests = new Map | OptimizedPackageManifest>() + async function _getCachedManifest(name: string) { + const cached = await manifestCache.get(name).catch(async e => { + if ((e as any).name !== 'SyntaxError') { + throw e + } + + getLogger().debug(`Removing corrupted cached manifest: ${name}`, e) + await manifestCache.delete(name) + }) + + return cached + } + + function getCachedManifest(name: string) { + if (manifests.has(name)) { + return manifests.get(name)! + } + + const cached = _getCachedManifest(name) + manifests.set(name, cached) + + return cached + } + + function setCachedManifest(name: string, manifest: PackageManifest) { + const opt = optimizePackageManfiest(manifest) + manifests.set(name, opt) + return manifestCache.set(name, opt) + } + + async function _getPackageJson(name: string, version: string): Promise { + // We don't need to fetch the entire manifest if we already + // have the requested version + const cached = await getCachedManifest(name) + const cachedPkgData = cached?.versions[version] + if (cachedPkgData) { + if ('symbolTable' in cached) { + return hydratePackageJson(name, cached.symbolTable, cachedPkgData) + } + + return cachedPkgData as PublishedPackageJson + } + + const m = await getPackageManifest(name) + const pkgData = m.versions[version] + if (!pkgData) { + throw new Error(`Package not found: ${name}@${version}`) + } + + if ('symbolTable' in m) { + return hydratePackageJson(name, m.symbolTable, pkgData) + } + + return pkgData as PublishedPackageJson + } + + const getPackageJson = keyedMemoize(async (name: string, version: string) => { + const pkg = await _getPackageJson(name, version) + if (!pkg) { + throw new Error(`Package not found: ${name}@${version}`) + } + return pkg + }) + + interface ETagManifest { + [name: string]: { + eTag?: string + requestCacheExpirationTime?: number + } + } + + const etagsKey = '__etags__' + const getEtags = memoize(async function () { + return (await manifestCache.get(etagsKey)) ?? {} + }) + + async function setEtag(name: string, value: string, maxAge?: number) { + const tags = await getEtags() + const paddedMaxAge = (maxAge ?? 0) + 300 // We'll cache for just a little bit longer + tags[name] = { + eTag: value, + requestCacheExpirationTime: Date.now() + (paddedMaxAge * 1000), + } + } + + async function close() { + cancelManifestRequests() + const tags = await getEtags() + await manifestCache.set(etagsKey, tags) + } + + async function _getPackageManifest(name: string) { + const etags = await getEtags() + const cacheTime = etags[name]?.requestCacheExpirationTime + if (cacheTime && Date.now() <= cacheTime) { + const cached = await getCachedManifest(name) + if (cached) { + return cached + } + } + + let resp = await client.getPackageManifest(name, etags[name]?.eTag, manifestAbortController) + if (!resp.manifest) { + const cached = await getCachedManifest(name) + if (cached) { + if (resp.maxAge) { + await setEtag(name, etags[name]!.eTag!, resp.maxAge) + } + + return cached + } + + resp = (await client.getPackageManifest(name, undefined, manifestAbortController)) + } + + if (!resp.manifest) { + throw new Error(`Missing manifest from package registry response. Package name: ${name}`) + } + + // I've seen this happen once or twice + if (!resp.manifest.versions) { + throw new Error(`Corrupted package manifest for package "${name}": ${JSON.stringify(resp.manifest, undefined, 4)}`) + } + + if (resp.etag) { + await setEtag(name, resp.etag, resp.maxAge) + } + + await setCachedManifest(name, resp.manifest) + + return resp.manifest + } + + const getPackageManifest = keyedMemoize(_getPackageManifest) + + async function listVersions(name: string) { + const m = await getPackageManifest(name) + + return Object.keys(m.versions) + } + + async function listTags(name: string) { + const m = await getPackageManifest(name) + const tags = 'symbolTable' in m ? m.tags : m['dist-tags'] + + return tags + } + + return { + close, + listTags, + listVersions, + getPackageJson, + + cancelError, + getPackageManifest, + } +} + +const npmUrl = 'https://registry.npmjs.org/' +const jsrUrl = 'https://npm.jsr.io' + +// TODO: add direct support for JSR (if people want it) +// Honestly I'm not so sure about JSR. Looks like they're reinventing the wheel... +// +// https://jsr.io/@/// +// https://jsr.io/@//meta.json +// https://jsr.io/@//_meta.json + + +// https://registry.npmjs.org/@aws-sdk/client-s3 + +type RegistryClient = ReturnType +export function createNpmRegistryClient(registryUrl = npmUrl, authToken?: string) { + const request = createRequester(registryUrl) + + function getHeaders() { + return { 'authorization': `Bearer ${Buffer.from(authToken!).toString('base64')}` } + } + + async function getPackageManifest(name: string, etag?: string, abortController?: AbortController): Promise<{ etag?: string; manifest?: PackageManifest; maxAge?: number } > { + const opt: any = { etag, acceptGzip: true, abortController, headers: authToken ? getHeaders() : undefined } + const res = await request(`GET /${name}`, undefined, undefined, opt) + + return { etag: opt.etag, manifest: res, maxAge: opt.maxAge } + } + + async function downloadPackage(tarballUrl: string): Promise { + return request(`GET ${tarballUrl}`, undefined, true, { headers: authToken ? getHeaders() : undefined }) + } + + async function getSigningKeys(): Promise { + return request(`GET /-/npm/v1/keys`, undefined, undefined, { headers: authToken ? getHeaders() : undefined }) + } + + return { getPackageManifest, downloadPackage } +} + +function getUrl(packageName: string, config: NpmConfig): string { + if (!packageName.startsWith('@')) { + return npmUrl + } + + const [scope] = packageName.split('/') + + return config.registries[scope] ?? npmUrl +} + +// TODO: make caching work with multiple registries (manifests + packages) +export function createMultiRegistryClient(dir: string): RegistryClient { + const clients = new Map() + + const getConfig = memoize(() => resolveNpmConfigs(dir)) + + + function getOrCreateClient(url: string, config: NpmConfig | undefined) { + const client = clients.get(url) + if (client) { + return client + } + + const hostname = url.replace(/^https?:\/\//, '') + const newClient = createNpmRegistryClient(url, config?.scopedConfig[hostname]?._authToken) + clients.set(url, newClient) + + return newClient + } + + async function getPackageManifest(name: string, etag?: string, abortController?: AbortController) { + const config = await getConfig() + const url = !config ? npmUrl : getUrl(name, config) + + return getOrCreateClient(url, config).getPackageManifest(name, etag, abortController) + } + + const getUrls = memoize(async () => { + const config = await getConfig() + const set = new Set([npmUrl]) + for (const k of Object.keys(config?.registries ?? {})) { + set.add(k) + } + for (const k of Object.keys(config?.scopedConfig ?? {})) { + set.add(`https://${k}`) + } + return set + }) + + async function findClient(tarballUrl: string) { + const config = await getConfig() + for (const k of await getUrls()) { + if (tarballUrl.startsWith(k)) { + return getOrCreateClient(k, config) + } + } + return getOrCreateClient(npmUrl, config) + } + + async function downloadPackage(tarballUrl: string) { + const client = await findClient(tarballUrl) + + return client.downloadPackage(tarballUrl) + } + + return { getPackageManifest, downloadPackage } +} + +const algs = ['sha256', 'sha512'] as const +type Alg = (typeof algs)[number] + +export function assertAlg(s: string): asserts s is Alg { + if (!algs.includes(s as any)) { + throw new Error(`Invalid algorithm "${s}". Must be one of: ${algs.join(', ')}`) + } +} + +interface GetPackageDataResponse { + readonly data: Buffer + readonly format: '.tar.gz' + readonly checksum: { + readonly alg: Alg + readonly value: string + } +} + +interface GetPackageSignatureResponse { + +} + +// function parseIntegrityString(integrity: string): GetPackageDataResponse['checksum'] { +// const match = integrity.match(/^([^-]+)-([^-]+)$/) +// if (!match) { +// throw new Error(`Failed to parse integrity string: ${integrity}`) +// } + +// const [_, alg, value] = match +// assertAlg(alg) + +// return { alg, value } +// } \ No newline at end of file diff --git a/src/pm/packageJson.ts b/src/pm/packageJson.ts new file mode 100644 index 0000000..864eab0 --- /dev/null +++ b/src/pm/packageJson.ts @@ -0,0 +1,460 @@ +import * as path from 'node:path' +import { homedir } from 'node:os' +import { getLogger } from '..' +import { getPreviousProgramFs, getProgramFs } from '../artifacts' +import { Fs, readDirectorySync } from '../system' +import { getHash, throwIfNotFileNotFoundError, tryReadJson } from '../utils' +import { SynapseConfiguration, getWorkingDir } from '../workspaces' +import { getBuildTarget, getFs } from '../execution' +import { providerPrefix } from '../runtime/loader' + +export interface PackageJson { + readonly os?: string[] + readonly cpu?: string[] + readonly name: string + readonly type?: 'module' + readonly main?: string + readonly files?: string[] + readonly bin?: string | Record + readonly module?: string + readonly version?: string + readonly private?: boolean + readonly exports?: any // PackageExport + readonly imports?: Record // Value is the same as exports + readonly browser?: Record + readonly scripts?: Record + readonly workspaces?: string[] + readonly engines?: Record & { node?: string } + readonly dependencies?: Record + readonly devDependencies?: Record + readonly peerDependencies?: Record + readonly bundledDependencies?: Record + readonly optionalDependencies?: Record + readonly optionalDevDependencies?: Record + readonly peerDependenciesMeta?: Record + readonly synapse?: { + readonly config?: SynapseConfiguration + readonly tools?: Record + readonly dependencies?: Record + readonly providers?: Record + readonly binaryDependencies?: Record + readonly devTools?: Record + } +} + +const packageJsonCache = new Map() + +export interface ResolvedPackage { + readonly data: PackageJson + readonly directory: string +} + +export async function getPackageJson(fs: Fs, dir: string, recursive = true, stopAt?: string): Promise { + const key = `${dir}:${recursive}` + if (packageJsonCache.has(key)) { + return packageJsonCache.get(key)! + } + + let result: { directory: string; data: PackageJson } | undefined + try { + const data = JSON.parse(await fs.readFile(path.resolve(dir, 'package.json'), 'utf-8')) as PackageJson + result = { directory: dir, data } + } catch (e) { + throwIfNotFileNotFoundError(e) + + if (!recursive || path.dirname(dir) === (dir ?? stopAt)) { + result = undefined + } else { + result = await getPackageJson(fs, path.dirname(dir), recursive, stopAt) + } + } + + packageJsonCache.set(key, result) + + return result +} + +export async function getImmediatePackageJsonOrThrow(dir = getWorkingDir()) { + const fs = getFs() + const pkg = await getPackageJson(fs, dir) + if (!pkg) { + throw new Error(`No "package.json" found: ${dir}`) + } + return pkg +} + +const previousPkgs = new WeakMap | Pick, PackageJson | undefined>() +const cachedPkgs = new WeakMap | Pick, PackageJson | undefined>() +export function setCompiledPkgJson(compiled: PackageJson, fs: Pick = getProgramFs()) { + cachedPkgs.set(fs, compiled) + if (previousPkgs.has(fs)) { + return fs.writeFile(`[#packages]package.json`, JSON.stringify(compiled)) + } + + return fs.readFile('[#packages]package.json') + .catch(() => {}) + .then(d => previousPkgs.set(fs, d ? JSON.parse(Buffer.from(d).toString()) : undefined)) + .then(() => fs.writeFile(`[#packages]package.json`, JSON.stringify(compiled))) +} + +export function resetCompiledPkgJson(fs: Pick = getProgramFs()) { + const prev = previousPkgs.get(fs) + + return prev && fs.writeFile(`[#packages]package.json`, JSON.stringify(prev)) +} + +export function getCompiledPkgJson(fs: Pick = getProgramFs()): Promise | PackageJson | undefined { + if (cachedPkgs.has(fs)) { + return cachedPkgs.get(fs) + } + + return tryReadJson(fs, `package.json`) + .then(val => { + if (!previousPkgs.has(fs)) { + previousPkgs.set(fs, val) + } + cachedPkgs.set(fs, val) + + return val + }) +} + +export async function getCurrentPkg() { + const cwd = getBuildTarget()?.workingDirectory ?? process.cwd() + const compiled = await getCompiledPkgJson() + if (compiled) { + return { directory: cwd, data: compiled } + } + + return getPackageJson(getFs(), cwd, false) +} + +interface DepProps { + pattern: string + dev?: boolean +} + +export interface DepsDiff { + readonly added?: Record + readonly removed?: Record + readonly changed?: Record; to: Partial }> +} + +function gatherDeps(pkg: Pick): Record { + const deps: Record = {} + if (pkg.dependencies) { + for (const [k, v] of Object.entries(pkg.dependencies)) { + deps[k] = { pattern: v } + } + } + + if (pkg.devDependencies) { + for (const [k, v] of Object.entries(pkg.devDependencies)) { + deps[k] = { pattern: v, dev: true } + } + } + + return deps +} + +export function getPreviousPkg(fs = getProgramFs()) { + if (previousPkgs.has(fs)) { + return previousPkgs.get(fs) + } + + return getPreviousProgramFs().then(fs => fs ? getCompiledPkgJson(fs) : undefined) +} + +async function diffDeps(): Promise { + const [current, previous] = await Promise.all([ + getCompiledPkgJson(), + getPreviousPkg(), + ]) + + return diffPkgDeps(current, previous) +} + +export function diffPkgDeps( + current?: Pick, + previous?: Pick +): DepsDiff | undefined { + if (!current && !previous) { + return + } + + const currentDeps = current ? gatherDeps(current) : undefined + const previousDeps = previous ? gatherDeps(previous) : undefined + + if (currentDeps && !previousDeps) { + return { added: currentDeps } + } + + if (!currentDeps && previousDeps) { + return { removed: previousDeps } + } + + function diffProps(v: DepProps, p: DepProps) { + const devChanged = p.dev !== v.dev + const patternChanged = v.pattern !== p.pattern + if (!devChanged && !patternChanged) { + return + } + + return { + from: { + dev: devChanged ? p.dev : undefined, + pattern: patternChanged ? p.pattern : undefined, + }, + to: { + dev: devChanged ? v.dev : undefined, + pattern: patternChanged ? v.pattern : undefined, + }, + } + } + + const changed = new Map; to: Partial }>() + const result: DepsDiff = { added: {}, removed: {} } + for (const [k, v] of Object.entries(currentDeps!)) { + const p = previousDeps![k] + if (!p) { + result.added![k] = v + continue + } + + const change = diffProps(v, p) + if (change) { + changed.set(k, change) + } + } + + for (const [k, v] of Object.entries(previousDeps!)) { + if (changed.has(k)) continue + + const c = currentDeps![k] + if (!c) { + result.removed![k] = v + } + } + + return { + ...result, + changed: Object.fromEntries(changed), + } +} + +function isEmptyObject(obj?: Record) { + return !obj || Object.keys(obj).length === 0 +} + +function isEmptyDiff(diff: DepsDiff) { + return isEmptyObject(diff.added) && isEmptyObject(diff.removed) && isEmptyObject(diff.changed) +} + +export async function runIfPkgChanged(fn: () => Promise | T): Promise { + const diff = await diffDeps() + if (diff && isEmptyDiff(diff)) { + return + } + + getLogger().debug('Package has changed, checking installation', diff) + + return fn() +} + +// Git URLs +// ://[[:]@][:][:][/][# | #semver:] + +function parsePkgRequest(r: string): { name: string; version?: string; scheme?: string } { + const match = r.match(/^(?[a-z]+:)?(?@?[^@:\s]+)(?@[^@\s:\/]+)?(?#[0-9a-zA-Z\/\-]+)?$/) // Version constraint is fairly broad + if (!match?.groups) { + throw new Error(`Bad parse: ${r}`) + } + + const name = match.groups.name + const scheme = match.groups.scheme?.slice(0, -1) + + // GitHub e.g. `user/repo#feature/branch` + // This is the same format used by `npm` + if (!scheme && name.includes('/') && !name.startsWith('@')) { + if (isProbablyFilePath(name)) { + return { name } + } + + return { name, version: match.groups.commitish?.slice(1), scheme: 'github' } + } + + // FIXME: module specifiers should be able to include a scheme + return { name, version: match.groups.version?.slice(1), scheme: match.groups.scheme?.slice(0, -1) } +} + +function isProbablyFilePath(name: string) { + return name.startsWith('..') || name.startsWith('~') || name.startsWith('./') || name.startsWith('/') // TODO: windows backslash? +} + +export function resolveFileSpecifier(spec: string, workingDir = getWorkingDir()) { + const absPath = path.resolve(workingDir, spec.replace('~', homedir())) + const pkg = getFs().readFileSync(path.resolve(absPath, 'package.json'), 'utf-8') + if (!pkg) { + throw new Error(`Not a package: ${absPath}`) + } + + return { + specifier: JSON.parse(pkg).name ?? path.basename(absPath), + location: `file:${path.relative(workingDir, absPath)}`, + } +} + +export function parsePackageInstallRequests(requests: string[]): Record { + const result: Record = {} + for (const r of requests) { + const p = parsePkgRequest(r) + if (p.scheme === 'file' || isProbablyFilePath(p.name)) { + const resolved = resolveFileSpecifier(p.name) + result[resolved.specifier] = resolved.location + continue + } + + if (p.scheme && p.scheme !== 'npm') { + throw new Error(`Not implemented: ${p.scheme}`) + } + + result[p.name] = p.version ?? 'latest' + } + return result +} + +// https://github.com/oven-sh/bun/issues/3107 +function detectIndentLevel(jsonText: string) { + const firstNewLine = jsonText.indexOf('\n') + if (firstNewLine === -1) { + return 0 + } + + const secondNewLine = jsonText.indexOf('\n', firstNewLine) + const secondLine = jsonText.slice(firstNewLine+1, secondNewLine === -1 ? jsonText.length : secondNewLine) + + // meh + let indent = 0 + loop: for (let i = 0; i < secondLine.length; i++) { + switch (secondLine[i]) { + case '\t': + indent += 4 + continue + case ' ': + indent += 1 + continue + + default: + break loop + } + } + + return indent +} + +export function createSynapseProviderRequirement(name: string, constraint: string) { + return [`${providerPrefix}${name}`, `spr:_provider-${name}:${constraint}`] as const +} + +export function isFileUrl(version: string) { + return !!version.startsWith('file:') +} + +export function getRequired(pkg: Partial, includeOptional = true, includeDev = false, includeWorkspaces = false, includeSynapse = true) { + let isEmpty = true + const required: Record = {} + const optional = new Set() + + if (pkg.optionalDependencies) { + for (const [k, v] of Object.entries(pkg.optionalDependencies)) { + if (includeOptional) { + required[k] = v + isEmpty = false + } else { + optional.add(k) + } + } + } + + // if (pkg.peerDependenciesMeta) { + // for (const [k, v] of Object.entries(pkg.peerDependenciesMeta)) { + // if (v.optional && includeOptional) { + // optional.add(k) + // } + // } + // } + + if (pkg.synapse?.providers && includeSynapse) { + for (const [k, v] of Object.entries(pkg.synapse.providers)) { + const [name, pattern] = createSynapseProviderRequirement(k, v) + required[name] = pattern + isEmpty = false + } + } + + if (pkg.synapse?.dependencies && includeSynapse) { + for (const [k, v] of Object.entries(pkg.synapse.dependencies)) { + required[k] = `spr:${v}` + isEmpty = false + } + } + + if (pkg.dependencies) { + for (const [k, v] of Object.entries(pkg.dependencies)) { + if (!optional.has(k) && !isFileUrl(v)) { + required[k] = v + isEmpty = false + } + } + } + + if (includeDev && pkg.devDependencies) { + for (const [k, v] of Object.entries(pkg.devDependencies)) { + if (!optional.has(k) && !isFileUrl(v)) { + required[k] = v + isEmpty = false + } + } + } + + if (pkg.workspaces && includeWorkspaces) { + for (const [k, v] of Object.entries(resolveWorkspaces(pkg.workspaces))) { + required[k] = v + isEmpty = false + } + } + + return isEmpty ? undefined : required +} + + +export function resolveWorkspaces(workspaces: string[], workspaceRootDir?: string) { + const result: Record = {} + + function resolve(dir: string) { + const resolved = resolveFileSpecifier(dir, workspaceRootDir) + result[resolved.specifier] = resolved.location + } + + for (const v of workspaces) { + const isPattern = v.endsWith('*') || v.endsWith('/') + if (!isPattern) { + // throw new Error(`Pattern not implemented: "${v}". A workspace pattern must end with "*"`) + + resolve(v) + continue + } + + const absPath = path.resolve(workspaceRootDir ?? getWorkingDir(), v) + const files = readDirectorySync(v.endsWith('/') ? absPath : path.dirname(absPath)) + const base = v.endsWith('/') ? '' : path.basename(v).slice(0, -1) + for (const f of files) { + if (f.type !== 'directory') continue + if (!f.name.startsWith(base)) continue + + resolve(path.resolve(v.endsWith('/') ? absPath : v.slice(0, -1), f.name)) + } + } + + return result +} \ No newline at end of file diff --git a/src/pm/packages.ts b/src/pm/packages.ts new file mode 100644 index 0000000..b7ea4f8 --- /dev/null +++ b/src/pm/packages.ts @@ -0,0 +1,4292 @@ +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import { TargetsFile, readPointersFile } from '../compiler/host' +import { getLogger, runTask } from '../logging' +import { synapsePrefix, providerPrefix } from '../runtime/loader' +import type { DependencyTree, PackageInfo } from '../runtime/modules/serdes' +import { SynapseConfiguration, getSynapseDir, getRootDirectory, getToolsDirectory, getUserSynapseDirectory, getWorkingDir, getWorkingDirectory, getGlobalCacheDirectory, resolveProgramBuildTarget, getPackageCacheDirectory } from '../workspaces' +import { Mutable, acquireFsLock, createHasher, createMinHeap, deepClone, escapeRegExp, getHash, gunzip, isNonNullable, isRelativeSpecifier, isRunningInVsCode, isWindows, keyedMemoize, memoize, resolveRelative, strcmp, throwIfNotFileNotFoundError, tryReadJson } from '../utils' +import type { TerraformPackageManifest } from '../runtime/modules/terraform' +import { ProviderConfig, createProviderGenerator, getProviderSource, listProviderVersions } from '../codegen/providers' +import { createHash } from 'node:crypto' +import { DataRepository, InstallationAttributes, ModuleBindingResult, ReadonlyBuildFs, Snapshot, commitPackages, getInstallation, getModuleMappings, getPrefixedPath, isSnapshotTarball, readInfraMappings, unpackSnapshotTarball, getDataRepository, getProgramFs, loadSnapshot, tryLoadSnapshot, commitProgram, getProgramHash } from '../artifacts' +import { addImplicitPackages, getDependentsData, getPackageOverride, installBin, setDependentsData, updateDependent } from './publish' +import { ModuleResolver } from '../runtime/resolver' +import { extractTarball, extractToDir } from '../utils/tar' +import { Fs, JsonFs, SyncFs, readDirectorySync } from '../system' +import { glob } from '../utils/glob' +import { getBuildTargetOrThrow, getFs, getSelfPath, getSelfPathOrThrow, isCancelled } from '../execution' +import { getTerraformPath } from '../deploy/deployment' +import { TypesFileData } from '../compiler/resourceGraph' +import { createPointer, isDataPointer, pointerPrefix, toAbsolute, toDataPointer } from '../build-fs/pointers' +import { ImportMap, SourceInfo, expandImportMap, flattenImportMap, hoistImportMap } from '../runtime/importMaps' +import { packages } from '@cohesible/resources' +import { execCommand } from '../utils/process' +import { PackageJson, ResolvedPackage, createSynapseProviderRequirement, diffPkgDeps, getCurrentPkg, getPackageJson, getRequired, isFileUrl, resolveFileSpecifier, resolveWorkspaces, runIfPkgChanged, setCompiledPkgJson } from './packageJson' +import { QualifiedBuildTarget, resolveBuildTarget, toNodeArch, toNodePlatform } from '../build/builder' +import { InstallSummary, createInstallView } from '../cli/views/install' +import { VersionConstraint, compareConstraints, isCompatible, isExact, parseVersionConstraint } from './versions' +import { createRequester, fetchData } from '../utils/http' +import { createToolRepo, toolPrefix } from './tools' +import { cleanDir, fastCopyDir, removeDir } from '../zig/fs-ext' +import { colorize, printLine } from '../cli/ui' +import { OptimizedPackageManifest, PackageManifest, PublishedPackageJson, createManifestRepo, createMultiRegistryClient, createNpmRegistryClient } from './manifests' +import { createGitHubPackageRepo, downloadGitHubPackage, githubPrefix } from './repos/github' + +// legacy +const providerRegistryHostname = '' + +export type ResourceKind = 'inline' | 'construct' + +export interface PackageEntry { + readonly name: string + readonly version: string // This can be a pattern e.g. `^2.0.0` +} + +// Exports can be conditional +// Key order matters for determining which export to use +// Common conditions: +// `node` +// `browser` +// `import` +// `require` +// `default` +// `module` +// `development` +// `production` + +interface SubpathExports { + readonly '.'?: PackageExport + readonly [entry: `./${string}`]: PackageExport +} + +export interface ConditionalExports { + readonly node?: PackageExport + readonly browser?: PackageExport + readonly require?: PackageExport + readonly import?: PackageExport + readonly module?: PackageExport + readonly default?: PackageExport + readonly [entry: string]: PackageExport | undefined +} + +type PackageExport = SubpathExports | ConditionalExports | string | null + +// Target is expected to be a relative path from the target package directory +export function resolveExport(target: string, exports: PackageExport, conditions?: string[], defaultEntrypoint?: string): string | [resolved: string, ...conditions: string[]] { + const condSet = new Set(conditions) + condSet.add('default') + + return inner(exports) + + function inner(exports: PackageExport, pathReplacement?: string) { + if (!exports) { + throw new Error(`No exports available matching target "${target}": ${exports}`) + } + + if (typeof exports === 'string') { + if (pathReplacement !== undefined) { + return exports.replace(/\*/, pathReplacement) + } + + if (pathReplacement === undefined && target !== '.') { + throw new Error(`Unable to resolve subpath "${target}". Package does not export any subpaths.`) + } + + return exports + } + + if (Array.isArray(exports)) { + const errors: any[] = [] + for (const x of exports) { + try { + return inner(x, pathReplacement) + } catch (e) { + errors.push(e) + } + } + + throw new AggregateError(errors, 'Failed to resolve exports array') + } + + const keys = Object.keys(exports) + if (keys.length === 0) { + throw new Error(`Found empty exports object while resolving subpath "${target}"`) + } + + if (isRelativeSpecifier(keys[0]) || keys[0][0] === '#') { + if (pathReplacement !== undefined) { + throw new Error(`Unexpected nested subpaths exports found while resolving "${target}"`) + } + + return resolveSubpathExports(exports as SubpathExports) + } else { + return resolveConditionalExports(exports as ConditionalExports, pathReplacement) + } + } + + function resolveSubpathExports(subpaths: SubpathExports): string | [resolved: string, ...conditions: string[]] { + if (defaultEntrypoint && target === '.' && !('.' in subpaths)) { + return defaultEntrypoint + } + + // Start from longest subpath first to handle private exports + // Paths containing a wildcard are considered longer + const entries: [string, PackageExport][] = Object.entries(subpaths).sort((a, b) => b[0].length - a[0].length) + for (const [k, v] of entries) { + const pattern = new RegExp(`^${escapeRegExp(k).replace(/\\\*/, '(.*)')}$`) + const match = pattern.exec(target) + if (match) { + const replacement = match?.[1] + if (replacement) { + return inner(v, replacement) + } + + return inner(v, '') + } + } + + throw new Error(`Found no subpaths matching "${target}": ${entries.map(x => x[0])}`) + } + + function resolveConditionalExports(cond: ConditionalExports, pathPattern?: string): [resolved: string, ...conditions: string[]] { + // We iterate over the condition set instead of + // the conditions to ensure priority + for (const k of condSet) { + const v = cond[k] + if (!v) continue + + try { + const resolved = inner(v, pathPattern) + if (typeof resolved === 'string') { + return [resolved, k] + } + return [resolved[0], ...resolved[1], k] + } catch {} + } + + throw new Error(`Found no conditions matching "${target}"`) + } +} + +function getConditions(mode: 'cjs' | 'esm' | 'synapse') { + switch (mode) { + case 'cjs': + return ['node', 'require'] + case 'esm': + return ['module', 'import'] + case 'synapse': + return ['module', 'import', 'node', 'require'] + } + + throw new Error(`Unknown mode: ${mode}`) +} + +// "synapse" mode can use both esm + cjs but prefers esm when available + +interface ResolveResult { + readonly fileName: string + readonly moduleType: 'cjs' | 'esm' +} + +export function resolvePrivateImport(spec: string, pkg: PackageJson, mode: 'cjs' | 'esm' | 'synapse'): ResolveResult { + const imports = pkg.imports + if (!imports) { + throw new Error(`Package "${pkg.name}" has no imports field`) + } + + const conditions = getConditions(mode) + const resolved = resolveExport(spec, imports, conditions) + + if (typeof resolved === 'string') { + return { + fileName: resolved, + moduleType: pkg.type === 'module' ? 'esm' : 'cjs', + } + } + + const lastCond = resolved[resolved.length - 1] + if (lastCond === 'node' && pkg.type === 'module') { + return { + fileName: resolved[0], + moduleType: 'esm', + } + } + + const moduleType = lastCond === 'import' || lastCond === 'module' ? 'esm' + : lastCond === 'node' || lastCond === 'require' ? 'cjs' + : lastCond === 'default' ? pkg.type === 'module' ? 'esm' : 'cjs' : 'cjs' + + return { + fileName: resolved[0], + moduleType, + } +} + +export function resolveBareSpecifier(spec: string, pkg: PackageJson, mode: 'cjs' | 'esm' | 'synapse'): ResolveResult { + const components = getSpecifierComponents(spec) + const target = components.export ? `./${components.export}` : '.' + const defaultEntrypoint = mode === 'cjs' ? pkg.main : pkg.module + const pkgType = pkg.type === 'module' ? 'esm' : 'cjs' + const defaultModuleType = mode === 'synapse' ? pkgType : mode + + if (pkg.exports) { + const conditions = getConditions(mode) + const resolved = resolveExport(target, pkg.exports, conditions, defaultEntrypoint) + if (typeof resolved === 'string') { + return { + fileName: resolved, + moduleType: mode === 'synapse' ? (resolved === pkg.main ? 'cjs' : resolved === pkg.module ? 'esm' : pkgType) : pkgType, + } + } + + const lastCond = resolved[resolved.length - 1] + const moduleType = lastCond === 'import' || lastCond === 'module' ? 'esm' + : lastCond === 'node' || lastCond === 'require' ? 'cjs' + : lastCond === 'default' ? pkgType : defaultModuleType + + return { + fileName: resolved[0], + moduleType, + } + } + + // Direct access to the package + if (components.export) { + return { fileName: target, moduleType: defaultModuleType } + } + + if (mode === 'cjs') { + if (!defaultEntrypoint) { + // throw new Error(`Package "${spec}" has no main export`) + return { fileName: 'index.js', moduleType: 'cjs' } + } + + return { fileName: defaultEntrypoint, moduleType: 'cjs' } + } + + if (pkg.module) { + return { fileName: pkg.module, moduleType: 'esm' } + } + + if (!pkg.main) { + throw new Error(`Package "${spec}" has no module or main export`) + } + + return { fileName: pkg.main, moduleType: defaultModuleType } +} + +// From https://github.com/nodejs/node/blob/a00f0b1f0a53d47ae334b67e4566f27be02d7d8a/lib/internal/modules/esm/resolve.js#L346 +const invalidSegmentRegEx = /(^|\\|\/)((\.|%2e)(\.|%2e)?|(n|%6e|%4e)(o|%6f|%4f)(d|%64|%44)(e|%65|%45)(_|%5f)(m|%6d|%4d)(o|%6f|%4f)(d|%64|%44)(u|%75|%55)(l|%6c|%4c)(e|%65|%45)(s|%73|%53))?(\\|\/|$)/i +const invalidPackageNameRegEx = /^\.|%|\\/ + +const packageCache = new Map() +const packageLockJsonCache = new Map() + + +export interface PackageLockV3Entry { + name?: string + version?: string + resolved?: string + integrity?: string + link?: boolean + dev?: boolean + bin?: Record + optional?: boolean + engines?: Record // e.g. { "node": ">=14.17" } + dependencies?: Record + devDependencies?: Record + peerDependencies?: Record + optionalDependencies?: Record +} + +export interface PackageLockV3 { + name: string + lockfileVersion: 3 + packages: Record +} + +// This function doesn't look at parent directories unlike `getPackageJson` +export async function getPackageLockJson(fs: Fs, dir: string): Promise< { directory: string; data: PackageLockV3 } | undefined> { + if (packageLockJsonCache.has(dir)) { + return packageLockJsonCache.get(dir)! + } + + let result: { directory: string; data: PackageLockV3 } | undefined + try { + const data = JSON.parse(await fs.readFile(path.resolve(dir, 'package-lock.json'), 'utf-8')) as PackageLockV3 + result = { directory: dir, data } + } catch (e) { + throwIfNotFileNotFoundError(e) + + // Best effort look up + const dotPackageLock = path.resolve(dir, 'node_modules', '.package-lock.json') + const contents = await fs.readFile(dotPackageLock, 'utf-8').catch(throwIfNotFileNotFoundError) + + if (contents !== undefined) { + const data = JSON.parse(contents) as PackageLockV3 + result = { directory: dir, data } + } + } finally { + packageLockJsonCache.set(dir, result) + } + + return result +} + +export async function getNearestPackageLock(fs: Fs, dir: string): Promise< { directory: string; data: PackageLockV3 }> { + let currentDir = dir + while (true) { + const result = await getPackageLockJson(fs, currentDir) + if (result) { + return result + } + + const nextDir = path.dirname(currentDir) + if (nextDir === currentDir) { + break + } + + currentDir = nextDir + } + + throw new Error(`Unable to find a package lock file starting from: ${dir}`) +} + +function splitAt(str: string, pos: number): [string, string] { + return [str.slice(0, pos), str.slice(pos + 1)] +} + +interface SpecifierComponents { + readonly scheme?: string + readonly scope?: string // authority + readonly name: string + readonly export?: string +} + +export function getSpecifierComponents(specifier: string): SpecifierComponents { + const schemeIndex = specifier.indexOf(':') + if (schemeIndex !== -1) { + const [scheme, remainder] = splitAt(specifier, schemeIndex) + const components = getSpecifierComponents(remainder) + if (components.scheme) { + throw new Error(`Malformed module specifier: only one colon (:) is allowed: ${specifier}`) + } + + return { scheme, ...components } + } + + const isScoped = specifier.startsWith('@') + if (isScoped) { + const [scope, spec, ...rest] = specifier.split('/') + + return { + scope, + name: `${scope}/${spec}`, + export: rest.length > 0 ? rest.join('/') : undefined, + } + } + + const [spec, ...rest] = specifier.split('/') + + return { + name: spec, + export: rest.length > 0 ? rest.join('/') : undefined, + } +} + +// https://docs.npmjs.com/policies/crawlers +// https://github.com/npm/concurrent-couch-follower +// https://skimdb.npmjs.com/registry + +// Have to use CouchbaseDB to sync w/ the npm registry + +interface ResolvedDependency { + name: string + version: string + peers: Record + dependencies: Record +} + +interface ConstraintGroup { + readonly id: number + readonly name: string + readonly constraint: VersionConstraint + readonly parents?: ConstraintGroup[] +} + +export type ResolvePatternResult = { name: string, version: VersionConstraint } + +function isTag(version: string) { + return !!version.match(/^[A-Za-z]/) +} + +export interface PackageRepository { + listVersions(name: string): Promise | string[] + getDependencies(name: string, version: string): Promise | undefined> + resolvePattern(spec: string, pattern: string): Promise | ResolvePatternResult + getPeerDependencies?(name: string, version: string): Promise | undefined> + + getPackageJson(name: string, version: string): Promise +} + + + +type ImportMap2 = ImportMap + +const getNpmPackageRepo = memoize(() => { + return createNpmPackageRepo() +}) + +const getMultiRegistryNpmClient = keyedMemoize(createMultiRegistryClient) + +interface NpmRepoOptions { + shouldPrefetch?: boolean + system?: { cpu: string; os: string } +} + +interface NpmPackageRepository extends PackageRepository { + close: () => Promise + maybeDownloadPackage: (url: string, dest: string) => Promise<{ cached: boolean; dest: string }> + isDownloading: (dest: string) => boolean +} + +function createNpmPackageRepo(opt?: NpmRepoOptions): NpmPackageRepository { + const client = createNpmRegistryClient() + const manifestRepo = createManifestRepo(client) + const packagesDir = path.resolve(getUserSynapseDirectory(), 'packages') + + const { shouldPrefetch = true } = opt ?? {} + + const prefetched = new Set() + + async function prefetchDeps(name: string, manifest: PackageManifest | OptimizedPackageManifest): Promise { + if (prefetched.has(name)) { + return + } + + prefetched.add(name) + + const tags = 'symbolTable' in manifest ? manifest.tags : manifest['dist-tags'] + const latest = tags?.['latest'] ?? Object.keys(manifest.versions).pop()! + const deps = getRequired(await getPackageJson(name, latest), undefined, undefined, undefined, false) ?? {} + for (const [k, v] of Object.entries(deps)) { + const p = resolvePattern(k, v) + if (p instanceof Promise) { + return p.then(r => manifestRepo.getPackageManifest(r.name).then(m => prefetchDeps(r.name, m))).catch(e => { + if (e !== manifestRepo.cancelError) { throw new Error(`Failed to prefetch "${k}@${v}"`, { cause: e }) } + }) + } else { + return manifestRepo.getPackageManifest(p.name).then(m => prefetchDeps(p.name, m)).catch(e => { + if (e !== manifestRepo.cancelError) { throw new Error(`Failed to prefetch "${k}@${v}"`, { cause: e }) } + }) + } + } + } + + async function getPackageJson(name: string, version: string) { + const pkg = await manifestRepo.getPackageJson(name, version).catch(e => { + throw new Error(`Failed to get package "${name}@${version}"`, { cause: e }) + }) + + return pkg + } + + async function close() { + await manifestRepo.close() + } + + async function resolveTag(name: string, tag: string): Promise { + const tags = await manifestRepo.listTags(name) + const v = tags?.[tag] + if (!v) { + throw new Error(`No version found matching tag "${tag}" for package: ${name}`) + } + + return parseVersionConstraint(v) + } + + async function listVersions(name: string) { + return manifestRepo.listVersions(name) + } + + async function getDependencies(name: string, version: string) { + return getRequired(await getPackageJson(name, version)) + } + + async function getPeerDependencies(name: string, version: string) { + const pkg = await getPackageJson(name, version) + const peers: Record = { ...pkg.peerDependencies } + + let isEmpty = true + if (pkg.peerDependencies) { + for (const [k, v] of Object.entries(pkg.peerDependencies)) { + if (!isFileUrl(v) && !pkg.peerDependenciesMeta?.[k]?.optional) { + peers[k] = v + isEmpty = false + } + } + } + + return !isEmpty ? peers : undefined + } + + function _resolvePattern(spec: string, pattern: string): Promise | ResolvePatternResult { + // "npm:string-width@^4.2.0" + if (pattern.startsWith('npm:')) { + const rem = pattern.slice(4) + + // "npm:@docusaurus/react-loadable@6.0.0" + if (rem.startsWith('@')) { + const [name, version = '*'] = rem.slice(1).split('@') + + return { name: `@${name}`, version: parseVersionConstraint(version) } + } + + const [name, version = '*'] = rem.split('@') + + return { name, version: parseVersionConstraint(version) } + } + + if (!isTag(pattern)) { + return { name: spec, version: parseVersionConstraint(pattern) } + } + + return resolveTag(spec, pattern).then(version => ({ name: spec, version })) + } + + function resolvePattern(spec: string, pattern: string): Promise | ResolvePatternResult { + if (!shouldPrefetch) { + return _resolvePattern(spec, pattern) + } + + const res = _resolvePattern(spec, pattern) + if (res instanceof Promise) { + return res.then(resp => { + manifestRepo.getPackageManifest(resp.name).then(m => prefetchDeps(resp.name, m).catch(e => getLogger().debug(`Failed to prefetch "${resp.name}"`, e))) + + return resp + }) + } + + manifestRepo.getPackageManifest(res.name).then(m => prefetchDeps(res.name, m).catch(e => getLogger().debug(`Failed to prefetch "${res.name}"`, e))) + return res + } + + async function downloadPackage(url: string, dest: string) { + const fs = getFs() + const tarball = await client.downloadPackage(url) + const files = extractTarball(tarball) + const pkgFiles = files + .filter(f => f.path === 'package.json' || !!f.path.match(/[\\\/]package\.json$/)) + .sort((a, b) => a.path.length - b.path.length) + + const root = pkgFiles[0]?.path + if (!root) { + throw new Error(`Failed to find package.json in tarball`) + } + + const prefix = root.replace(/package\.json$/, '') + await Promise.all(files.map(async f => { + if (!f.path.startsWith(prefix)) { + return + } + + const absPath = path.resolve(dest, f.path.slice(prefix.length)) + if (f.contents.length > 0 && !f.path.endsWith('/')) { + await fs.writeFile(absPath, f.contents, { mode: f.mode }) + } else { + // FIXME: set dir permissions + // This is a directory + } + })) + } + + const pending = new Map>() + + function maybeDownloadPackage(url: string, dest: string) { + if (pending.has(dest)) { + return pending.get(dest)! + } + + const p = (async function () { + // TODO: need to place a tmp file while we extract in case + // we crash while extracting + if (await getFs().fileExists(dest)) { + return { cached: true, dest } + } + + await downloadPackage(url, dest) + + return { cached: false, dest } + })() + + pending.set(dest, p) + + return p + } + + function isDownloading(dest: string) { + return pending.has(dest) + } + + return { listVersions, getDependencies, getPeerDependencies, resolvePattern, getPackageJson, maybeDownloadPackage,isDownloading, close } +} + +function parseCspmRef(url: string) { + const ident = url.slice(url.startsWith('cspm') ? 5 : 4) + if (!ident.startsWith('#')) { + throw new Error(`Public packages not implemented`) + } + + return ident.slice(1) +} + +function createFilePackageRepo(fs: Fs, workingDirectory: string): PackageRepository { + const _getSnapshot = keyedMemoize((dir: string) => fs.readFile(path.resolve(dir, '.synapse', 'snapshot.json'), 'utf-8') + .then(JSON.parse, throwIfNotFileNotFoundError) as Promise + ) + + const _getPackageJson = keyedMemoize((dir: string) => { + // const snapshot = await _getSnapshot(dir) + // if (!snapshot) { + // return getPackageJson(fs, dir, false) + // } + + // const store = await getDataRepository().getBuildFs(snapshot.storeHash) + // if (store.index.files['package.json']) { + // return getDataRepository().readData(store.index.files['package.json'].hash).then(d => JSON.parse(Buffer.from(d).toString('utf-8'))) + // } + + return getPackageJson(fs, dir, false) + }) + + async function listVersions(name: string) { + const pkgDir = path.resolve(workingDirectory, name) + const pkg = await _getPackageJson(pkgDir) + if (!pkg) { + throw new Error(`No package found for path: ${name} [from ${workingDirectory}]`) + } + + const version = pkg.data.version ?? '0.0.1' + + return [version] + } + + async function resolvePattern(spec: string, pattern: string): Promise { + const pkgDir = path.resolve(workingDirectory, pattern) + const pkg = await _getPackageJson(pkgDir) + if (!pkg) { + throw new Error(`No package found for path: ${pattern} [from ${workingDirectory}]`) + } + + return { + name: pkg.data.name ?? spec, + version: parseVersionConstraint(pkg.data.version ?? '*'), + } + } + + async function getDependencies(name: string, version: string) { + const pkgDir = path.resolve(workingDirectory, name) + const pkg = await _getPackageJson(pkgDir) + if (!pkg) { + throw new Error(`No package found for path: ${name} [from ${workingDirectory}]`) + } + + return getRequired(pkg.data) + } + + return { + listVersions, + resolvePattern, + getDependencies, + getPackageJson: async (name, version) => { + const pkgDir = path.resolve(workingDirectory, name) + const pkg = await _getPackageJson(pkgDir) // TODO: use the compiled `package.json` if present + if (!pkg) { + throw new Error(`No package found for path: ${name} [from ${workingDirectory}]`) + } + + const snapshot = await _getSnapshot(pkgDir) + if (snapshot) { + return { + ...pkg.data, + version, + dist: { + tarball: pkgDir, + integrity: snapshot.storeHash, + isSynapsePackage: true, + }, + } + } + + const bt = await resolveProgramBuildTarget(pkgDir).catch(() => {}) + if (!bt) { + return { + ...pkg.data, + version, + dist: { tarball: pkgDir, integrity: '' }, + } + } + + + const integrity = await getProgramHash(bt.programId, getDataRepository(undefined, bt.buildDir)) + + return { + ...pkg.data, + version, + dist: { + tarball: pkgDir, + integrity: integrity ?? '', + isSynapsePackage: !!integrity, + }, + } + } + } +} + +function createSprRepoWrapper( + fs: Fs, + projectId: string, + workingDirectory: string +): PackageRepository & { close: () => Promise } { + const manifestCacheDir = path.resolve(getGlobalCacheDirectory(), 'package-manifests') + const sprCacheDir = path.resolve(manifestCacheDir, 'spr') + const npmRepo = getNpmPackageRepo() + const fileRepo = createFilePackageRepo(fs, workingDirectory) + const toolRepo = createToolRepo() + const githubRepo = createGitHubPackageRepo() + + const _getTerraformPath = memoize(getTerraformPath) + async function _getProviderVersions(name: string) { + // Double reverse! + return listProviderVersions(name, providerRegistryHostname, await _getTerraformPath()).then(x => x.reverse()) + } + + const workspacePackages = new Map() + + const getPrivatePackageManifest = keyedMemoize(async function (name: string): Promise { + const l = path.resolve(sprCacheDir, name) + + try { + return JSON.parse(await fs.readFile(l, 'utf-8')) + } catch (e) { + throwIfNotFileNotFoundError(e) + + const manifest = await packages.client.findProjectPackage(name, projectId) + if (!manifest) { + throw new Error(`No package named "${name}" found in current project`) + } + await fs.writeFile(l, JSON.stringify(manifest)) + + return manifest + } + }) + + function maybeMatchProvider(name: string) { + const prefixLen = name.startsWith('cspm:') ? 5 : 4 + const providerMatch = name.slice(prefixLen).match(/^_provider-([a-z]+):(.+)$/) + if (providerMatch) { + return { name: providerMatch[1], version: parseVersionConstraint(providerMatch[2]) } + } + } + + async function listVersions(name: string) { + if (name.startsWith(toolPrefix)) { + return toolRepo.listVersions(name.slice(toolPrefix.length)) + } + + if (name.startsWith(githubPrefix)) { + return githubRepo.listVersions(name.slice(githubPrefix.length)) + } + + if (name.startsWith('file:')) { + return fileRepo.listVersions(name.slice('file:'.length)) + } + + if (name.startsWith(providerPrefix)) { + return _getProviderVersions(name.slice(providerPrefix.length)) + } + + if (name.startsWith('cspm:')) { + const manifest = await getPrivatePackageManifest(parseCspmRef(name)) + + return Object.keys(manifest.versions) + } + + return npmRepo.listVersions(name) + } + + async function getPackageJson(name: string, version: string): Promise { + if (name.startsWith(toolPrefix)) { + return toolRepo.getPackageJson(name.slice(toolPrefix.length), version) + } + + if (name.startsWith(githubPrefix)) { + return githubRepo.getPackageJson(name.slice(githubPrefix.length), version) + } + + if (name.startsWith('file:')) { + return fileRepo.getPackageJson(name.slice('file:'.length), version) + } + + if (name.startsWith(providerPrefix)) { + return { + name: name.slice(providerPrefix.length), + version, + dist: { + tarball: getProviderSource(name.slice(providerPrefix.length)), + integrity: version, // TODO: use hash + }, + } + } + + if (name.startsWith('cspm:')) { + const manifest = await getPrivatePackageManifest(parseCspmRef(name)) + const pkgJson = manifest.versions[version] + + return { + ...pkgJson, + name: manifest.name, + dist: { + tarball: manifest.name, // XXX + integrity: pkgJson.packageDataHash, // XXX + } + } + } + + return npmRepo.getPackageJson(name, version) + } + + async function resolvePattern(spec: string, pattern: string) { + if (pattern.startsWith(toolPrefix)) { + return toolRepo.resolvePattern(spec, pattern) + } + + if (pattern.startsWith('file:')) { + if (pattern.endsWith('.tgz')) { + const location = path.resolve(workingDirectory, pattern.slice(5)) + const data = Buffer.from(await fs.readFile(location)) + const dest = path.resolve(getPackageCacheDirectory(), 'extracted', getHash(data)) + + if (!(await fs.fileExists(dest))) { + getLogger().log(`Extracting specifier "${spec}" to:`, dest) + const files = extractTarball(await gunzip(data)) + await Promise.all(files.map(async f => { + const absPath = path.resolve(dest, f.path) + await fs.writeFile(absPath, f.contents) + })) + } + + pattern = `file:${dest}` + } + + const resolved = await fileRepo.resolvePattern(spec, pattern.slice(5)) + workspacePackages.set(resolved.name, { version: resolved.version, location: pattern.slice(5) }) + + return { name: pattern, version: resolved.version } + } + + if (pattern.startsWith('cspm:') || pattern.startsWith('spr:')) { + const provider = maybeMatchProvider(pattern) + if (provider) { + return { name: `${providerPrefix}${provider.name}`, version: provider.version } + } + + const override = await getPackageOverride(parseCspmRef(pattern)) + if (override) { + return { name: `file:${override}`, version: parseVersionConstraint('*') } + } + + return { name: pattern, version: parseVersionConstraint('*') } + } + + // https://github.com/oven-sh/bun/issues/10889 + if (workspacePackages.has(spec)) { + const resolved = workspacePackages.get(spec)! + const parsed = parseVersionConstraint(pattern) + if (isCompatible(resolved.version, parsed)) { + return { + name: `file:${resolved.location}`, + version: parsed, + } + } + } + + // FIXME: this detection is fragile + if (pattern.startsWith(githubPrefix) || (pattern.includes('/') && !pattern.includes(':'))) { + const trimmed = pattern.startsWith(githubPrefix) ? pattern.slice(githubPrefix.length) : pattern + const resolved = await githubRepo.resolvePattern(spec, trimmed) + + return { + name: pattern.startsWith(githubPrefix) ? pattern : `github:${pattern}`, + version: resolved.version, + } + } + + return npmRepo.resolvePattern(spec, pattern) + } + + async function getDependencies(name: string, version: string) { + if (name.startsWith(toolPrefix)) { + return toolRepo.getDependencies(name.slice(toolPrefix.length), version) + } + + if (name.startsWith(githubPrefix)) { + return githubRepo.getDependencies(name.slice(githubPrefix.length), version) + } + + if (name.startsWith('file:')) { + return fileRepo.getDependencies(name.slice('file:'.length), version) + } + + if (name.startsWith(providerPrefix)) { + return + } + + if (name.startsWith('cspm:')) { + const manifest = await getPrivatePackageManifest(parseCspmRef(name)) + const inst = manifest.versions[version] + if (!inst) { + throw new Error(`Version "${version}" not found for package: ${name}`) + } + + return getRequired(inst.packageFile) + } + + return npmRepo.getDependencies(name, version) + } + + async function getPeerDependencies(name: string, version: string) { + if (name.startsWith('file:')) { + return fileRepo.getPeerDependencies?.(name.slice('file:'.length), version) + } + + if (name.startsWith(githubPrefix)) { + return // TODO + } + + if (name.startsWith(providerPrefix)) { + return + } + + if (name.startsWith(toolPrefix)) { + return + } + + if (name.startsWith('cspm:')) { + // const manifest = await getPrivatePackageManifest(parseCspmRef(name)) + // const inst = manifest.versions[version] + // if (!inst) { + // throw new Error(`Version "${version}" not found for package: ${name}`) + // } + + // return getRequired(inst.packageFile) + return undefined // TODO + } + + return npmRepo.getPeerDependencies?.(name, version) + } + + return { listVersions, getDependencies, resolvePattern, getPeerDependencies, getPackageJson, close: npmRepo.close } +} + + +interface ResolveDepsResult { + readonly roots: Record + readonly installed: [ConstraintGroup, ResolvedDependency][] +} + +export async function resolveDeps(deps: Record, repo: PackageRepository): Promise { + interface Key { + next(id: number): Key + toString(): string + } + + function createPosKey(): Key { + const keys: Record = {} + + function inner(pos: number[], str = ''): Key { + function next(id: number) { + const nPos = [...pos, id].sort() + const k = nPos.join('.') + + return keys[k] ??= inner(nPos, k) + } + + function toString() { + return str + } + + return { next, toString } + } + + return inner([]) + } + + + type Element = [ + installed: [ConstraintGroup, ResolvedDependency][], + groups: ConstraintGroup[], + key: Key, + fScore: number, + ] + + // TODO: this should break ties by picking the most recent element first + const pq = createMinHeap((a, b) => a[3] - b[3]) + + const _listVersions = keyedMemoize((spec: string) => repo.listVersions(spec)) + const _resolvePattern = keyedMemoize((spec: string, pattern: string) => repo.resolvePattern(spec, pattern)) + + let groupCounter = 0 + const groups: ConstraintGroup[] = [] + const roots: Record = {} + + const patterns = await Promise.all(Object.entries(deps).map(async ([k, v]) => [k, await _resolvePattern(k, v)] as const)) + for (const [k, { name, version }] of patterns) { + const id = groupCounter++ + roots[k] = id + groups.push({ + id, + name, + constraint: version, + }) + } + + const scores = new Map() + pq.insert([[], groups, createPosKey(), 0]) + + while (pq.length > 0) { + const [installed, groups, key] = pq.extract() + if (groups.length === 0) { + return { roots, installed } + } + + if (isCancelled()) { + break + } + + const g = groups.pop()! + const nk = key.next(g.id) + const nks = nk.toString() + const s = scores.get(nks) + if (s !== undefined && installed.length + 1 >= s) { + continue + } + + async function resolveGroup(k: string) { + const ni = [...installed] + const ng: ConstraintGroup[] = [] + const r: ResolvedDependency = { + name: g.name, + version: k, + peers: {}, + dependencies: {}, + } + + async function resolve(spec: string, req: string, isPeer = false) { + const patternResult = await _resolvePattern(spec, req) + const name = patternResult.name + const constraint = patternResult.version + + // If an already installed group exists and has a compatible constraint then + // we can use it directly without changing anything. + const idx = ni.findIndex(x => x[0].name === name && isCompatible(x[0].constraint, constraint, true)) + if (idx !== -1) { + const ig = ni[idx] + const isBefore = compareConstraints(ig[0].constraint, constraint) < 0 + + if (isBefore) { + ni[idx] = [ + { + ...ig[0], + constraint, + parents: [g, ...(ig[0].parents ?? [])], + }, + ig[1] + ] + } else { + ni[idx] = [ + { ...ig[0], parents: [g, ...(ig[0].parents ?? [])], }, + ig[1] + ] + } + + if (!isPeer) { + r.dependencies[spec] = ig[0].id + } else { + r.peers[spec] = ig[0].id + } + + return true + } + + const eg = groups.find(x => x.name === name && isCompatible(x.constraint, constraint, true)) + if (eg) { + const isBefore = compareConstraints(eg.constraint, constraint) <= 0 + ng.push({ + id: eg.id, + name: eg.name, + constraint: isBefore ? constraint : eg.constraint, + parents: [g, ...(eg.parents ?? [])], + }) + } else { + if (isPeer) { + const isRootPeer = !g.parents || g.parents.length === 0 + const parents = isRootPeer + ? Object.values(roots).map(r => groups[r]) + : g.parents + + for (const p of parents) { + const alreadyInstalled = ni.findIndex(x => x[0] === p && (x[0].name === spec || x[1].peers[spec])) + if (alreadyInstalled !== -1) { + const z = ni[alreadyInstalled] + if (z[0].name === spec && !isCompatible(z[0].constraint, constraint)) { + return false + } + // TODO: implement this case + // } else if (z[1].peers[spec] && !isCompatible(z[1].peers[spec], constraint)) { + + // } + } + } + } + + ng.push({ + id: groupCounter++, + name, + constraint, + parents: [g] + }) + } + + if (!isPeer) { + r.dependencies[spec] = ng[ng.length - 1].id + } else { + r.peers[spec] = ng[ng.length - 1].id + } + + return true + } + + // Peer dependencies must be installed as a unit. They cannot be installed directly under the dependent. + // If they cannot be installed as a peer then the current dependency cannot be installed at all. + const peers = await repo.getPeerDependencies?.(g.name, k) + if (peers) { + for (const [k2, v2] of Object.entries(peers)) { + const didInstall = await resolve(k2, v2, true) + if (!didInstall) { + getLogger().warn(`Failed to install peer dependency: ${k2}`) + return + } + } + } + + const required = await repo.getDependencies(g.name, k) + if (required) { + for (const [k2, v2] of Object.entries(required)) { + await resolve(k2, v2) + } + } + + ni.push([g, r]) + + // const nk = key.next(g.id) + // const nks = nk.toString() + // const s = scores.get(nks) + const ns = ni.length + // if (s === undefined || ns < s) { + // Add any missing groups in + const ids = new Set(ng.map(x => x.id)) + for (const x of groups) { + if (!ids.has(x.id)) { + ng.push(x) + } + } + + scores.set(nks, ns) + pq.insert([ni, ng, nk, ns + ng.length]) + + // For UI + getLogger().emitInstallEvent({ phase: 'resolve', resolveCount: ns } as any) + + return true + //} + } + + // Fast path + if (isExact(g.constraint)) { + continue + } + + const versions = await _listVersions(g.name) + + let didAdd = false + for (let i = versions.length - 1; i >= 0; i--) { + const k = versions[i] + const c = parseVersionConstraint(k) + if (!isCompatible(g.constraint, c)) { + continue + } + + // Assumes `versions` is sorted + if (await resolveGroup(k)) { + didAdd = true + break + } + } + if (!didAdd) { + throw new Error(`No version found matching pattern "${g.constraint.source}" for package "${g.name}"`) + } + } + + throw new Error(`Failed to resolve packages`) +} + +// For every spec/constraint pair: +// 1. Search for an already installed version by walking up the tree +// 2. If none are found, install it in the first open slot (top-down) +// a) This may be immediately under the dependent +// 3. Repeat for every new install + +interface RootTree { + subtrees: Record // spec -> tree + parent?: undefined +} + +interface Tree { + id: number + name: string + constraint: VersionConstraint + subtrees: Record // spec -> tree + parent: Tree | RootTree + resolved?: ResolvedDependency + ghosts: Record +} + +interface RebuiltTree extends Tree { + subtrees: Record // spec -> tree + parent: RebuiltTree | RebuiltRootTree + dependencies: Record + peers: Record + resolved: ResolvedDependency +} + +interface RebuiltRootTree extends RootTree { + ids: number + subtrees: Record +} + +function rebuildTree(data: ReturnType): RebuiltRootTree | undefined { + const root: RebuiltRootTree = { ids: 0, subtrees: {} } + const r = data.mappings['#root'] + + const trees: Record = {} + function getTree(id: string): RebuiltTree { + const s = data.sources[id] + if (!s || s.type !== 'package') { + throw new Error(`No source found`) + } + + const deps = (s as any).data.dependencies ?? {} + const peers = (s as any).data.peers ?? {} + + const t: RebuiltTree = { + id: root.ids++, + name: s.data.name, + parent: root, + subtrees: {}, + constraint: parseVersionConstraint(s.data.version), + dependencies: {}, + peers: {}, + ghosts: {}, + resolved: { + name: s.data.type === 'synapse-provider' ? `${providerPrefix}${s.data.name}` : s.data.name, + version: s.data.version, + dependencies: deps, + peers: peers, + }, + } + trees[t.id] = t + + for (const [k, v] of Object.entries(data.mappings[id] ?? {})) { + t.subtrees[k] = getTree(v) + t.subtrees[k].parent = t + } + + return t + } + + try { + for (const [k, v] of Object.entries(r)) { + root.subtrees[k] = getTree(v) + } + + for (const t of Object.values(trees)) { + const ancestors: (RebuiltTree | RebuiltRootTree)[] = [] + let p = t + while (p) { + ancestors.push(p) + p = p.parent as any + } + + const deps = t.resolved.dependencies + for (const [k, v] of Object.entries(deps)) { + for (const p of ancestors) { + if (p.subtrees[k]) { + t.dependencies[k] = p.subtrees[k] + break + } + } + } + + const peers = t.resolved.peers + for (const [k, v] of Object.entries(peers)) { + for (const p of ancestors) { + if (p.subtrees[k]) { + t.peers[k] = p.subtrees[k] + break + } + } + } + } + } catch(e) { + getLogger().warn(`Failed to load existing installation data`, e) + return + } + + return root +} + +interface ResolveDepsOptions { + readonly oldData?: ReturnType + readonly validateResult?: boolean // default: false +} + +// The result is already hoisted +export async function resolveDepsGreedy(deps: Record, repo: PackageRepository, opt?: ResolveDepsOptions) { + const _listVersions = keyedMemoize((spec: string) => repo.listVersions(spec)) + const _resolvePattern = keyedMemoize((spec: string, pattern: string) => repo.resolvePattern(spec, pattern)) + const oldTreeRoot = opt?.oldData ? rebuildTree(opt.oldData) : undefined + + let ids = 0 + const root: RootTree = { subtrees: {} } + const oldTreeMap = new Map() + const patterns = Object.fromEntries(await Promise.all(Object.entries(deps).map(async ([k, v]) => [k, await _resolvePattern(k, v)] as const))) + for (const [k, v] of Object.entries(deps)) { + root.subtrees[k] = { + id: ids++, + name: patterns[k].name, + constraint: patterns[k].version, + subtrees: {}, + parent: root, + ghosts: {}, + } + + if (oldTreeRoot && oldTreeRoot.subtrees[k]) { + oldTreeMap.set(root.subtrees[k], oldTreeRoot.subtrees[k]) + } + } + + function findAlreadyInstalledTree(tree: Tree | RootTree, name: string, spec: string, constraint: VersionConstraint, ignoreTag?: boolean): Tree | false | undefined { + const g = !isRootTree(tree) ? tree.ghosts[spec] : undefined + const x = g ?? tree.subtrees[spec] + if (!x) { + if (!tree.parent) { + return + } + + return findAlreadyInstalledTree(tree.parent, name, spec, constraint, ignoreTag) + } + + + if (name !== x.name || !isCompatible(x.constraint, constraint, ignoreTag)) { + return false + } + + // The initial constraint was compatible but we chose an incompatible version + // This happens when a dependency has a stricter requirement than a previously + // added dependency. Example: + // + // semver - ^7.5.4 -> resolved 7.6.0 + // ...some other dep + // semver - 7.5.4 -> must resolve to 7.5.4 + // + if (x.resolved && !isCompatible(parseVersionConstraint(x.resolved.version), constraint, ignoreTag)) { + return false + } + + const isBefore = compareConstraints(x.constraint, constraint) < 0 + if (isBefore) { + x.constraint = constraint + } + + return x + } + + let lastClone: RootTree['subtrees'] | undefined + class ResolveFailure extends Error {} + + // Used for debugging + const originalParents = new Map() + function getOriginalParent(tree: Tree) { + return originalParents.get(tree) ?? tree.parent + } + + function printTree(tree: RootTree | Tree) { + if (isRootTree(tree)) { + printLine(``) + } else if (tree.resolved) { + printLine(`${tree.resolved.name}@${tree.resolved.version}`) + } else { + printLine(`${tree.name}@${tree.constraint.source} `) + } + } + + function printTrace(tree: Tree) { + let t: Tree | RootTree = tree + while (true) { + if (isRootTree(t)) { + break + } + + printTree(t) + t = getOriginalParent(t) + } + } + + async function resolveTree(tree: Tree, oldTree = oldTreeMap.get(tree), isTopFrame = false) { + if (tree.resolved) { + return + } + + const pendingTrees: Tree[] = [] + async function resolve(parent: Tree, spec: string, req: string, isPeer = false) { + const patternResult = await _resolvePattern(spec, req) + const name = patternResult.name + const constraint = patternResult.version + + return _resolve(parent, name, spec, constraint, isPeer) + } + + function _resolve(parent: Tree, name: string, spec: string, constraint: VersionConstraint, isPeer = false) { + const inst = findAlreadyInstalledTree(parent, name, spec, constraint, isPeer) + if (inst) { + let z = parent + const y = inst.parent + while (z !== y && z !== root as any) { + (z as any).ghosts[spec] = inst + z = z.parent as any + } + if (isPeer) { + parent.resolved!.peers[spec] = inst.id + } else { + parent.resolved!.dependencies[spec] = inst.id + } + + return inst + } + + // We have to add a new tree + const t: Tree = { + id: ids++, + name, + parent, // Temporary assignment + constraint, + subtrees: {}, + ghosts: {}, + } + originalParents.set(t, parent) + + // Not very accurate but it works + getLogger().emitInstallEvent({ phase: 'resolve', resolveCount: ids } as any) + + // Peer deps cannot be installed as immediate deps unless it's the root + const ancestors: (Tree | RootTree)[] = [] + let q: Tree | RootTree | undefined = isPeer ? parent.parent : parent + while (q) { + if (q.subtrees[spec] || (!isRootTree(q) && q.ghosts[spec])) break + ancestors.unshift(q) + q = q.parent + } + + const firstOpenTree = ancestors[0] + if (!firstOpenTree) { + // for (const [k, v] of Object.entries(root.subtrees)) { + // printTree(v) + // } + // if (root.subtrees[spec]) { + // printTrace(root.subtrees[spec]) + // } + // printTrace(parent) + if (isPeer) { + getLogger().warn(`Invalid peer dependency: ${spec}`) + //printLine(colorize('gray', ` invalid peer dependency: ${spec}`)) + const resolved = root.subtrees[spec] + parent.resolved!.peers[spec] = resolved.id + + return resolved + } + + throw new ResolveFailure(`Failed to resolve spec ${spec} ${constraint.source} [peer: ${isPeer}]`) + } + + for (const x of ancestors) { + if (!isRootTree(x)) { + x.ghosts[spec] = t + } + } + + t.parent = firstOpenTree + firstOpenTree.subtrees[spec] = t + pendingTrees.push(t) + + if (oldTree) { + const tt = isPeer ? oldTree.peers[spec] : oldTree.dependencies[spec] + if (tt) { + oldTreeMap.set(t, tt) + } + } + + if (isPeer) { + parent.resolved!.peers[spec] = t.id + } else { + parent.resolved!.dependencies[spec] = t.id + } + + return t + } + + async function resolveDeps(name: string, version: string) { + const peers = await repo.getPeerDependencies?.(name, version) + if (peers) { + for (const [k2, v2] of Object.entries(peers)) { + await resolve(tree, k2, v2, true) + } + } + + const required = await repo.getDependencies(name, version) + if (required) { + for (const [k2, v2] of Object.entries(required)) { + const p = await _resolvePattern(k2, v2) + _resolve(tree, p.name, k2, p.version) + } + } + + // Not using `Promise.all` here because it makes the result non-deterministic + for (const t of pendingTrees) { + await resolveTree(t) + } + } + + async function resolveOldTree(oldTree: RebuiltTree) { + tree.resolved = { + name: tree.name, + version: oldTree.resolved.version, + peers: {}, + dependencies: {}, + } + + for (const [k, v] of Object.entries(oldTree.peers)) { + _resolve(tree, v.resolved.name, k, parseVersionConstraint(v.resolved.version), true) + } + + for (const [k, v] of Object.entries(oldTree.dependencies)) { + _resolve(tree, v.resolved.name, k, parseVersionConstraint(v.resolved.version)) + } + + for (const t of pendingTrees) { + await resolveTree(t) + } + } + + function getClone() { + if (lastClone) { + for (const [k, v] of Object.entries(root.subtrees)) { + lastClone[k] = v + } + return lastClone + } + + return lastClone = { ...root.subtrees } + } + + // Really dumb/simple way to backtrack + let clonedSubtrees = isTopFrame ? getClone() : undefined + + function unwind(e: unknown) { + if (!(e instanceof ResolveFailure)) { + throw e + } + + delete (root as any).subtrees + ;(root as any).subtrees = clonedSubtrees + lastClone = undefined + clonedSubtrees = getClone() + } + + if (oldTree && isCompatible(parseVersionConstraint(oldTree.resolved.version), tree.constraint)) { + if (!isTopFrame) { + return resolveOldTree(oldTree) + } + + try { + return await resolveOldTree(oldTree) + } catch (e) { + unwind(e) + } + } + + // Fast path, allows us to skip `listVersions` + if (isExact(tree.constraint)) { + const v = tree.constraint.label + ? `${tree.constraint.pattern}-${tree.constraint.label}` + : tree.constraint.pattern + + tree.resolved = { + name: tree.name, + version: v, + peers: {}, + dependencies: {}, + } + + await resolveDeps(tree.name, v) + + return + } + + const versions = await _listVersions(tree.name) + + for (let i = versions.length - 1; i >= 0; i--) { + const k = versions[i] + const c = parseVersionConstraint(k) + if (!isCompatible(tree.constraint, c)) { + continue + } + + tree.resolved = { + name: tree.name, + version: k, + peers: {}, + dependencies: {}, + } + + // Assumes `versions` is sorted + await resolveDeps(tree.name, k) + break + } + + if (!tree.resolved) { + throw new ResolveFailure(`Failed to resolve dependency ${tree.name}@${tree.constraint.source}`) + } + } + + const v = Object.values(root.subtrees) + for (const t of v) { + await resolveTree(t, undefined, !!oldTreeRoot) + } + + if (opt?.validateResult) { + try { + await validateTree(root, repo) + } catch (e) { + await (repo as any).close?.() + throw e + } + } + + return root +} + +export function printTree(tree: Tree | RootTree, depth = 0, spec?: string) { + if (isRootTree(tree)) { + for (const [k, v] of Object.entries(tree.subtrees)) { + printTree(v, depth, k) + } + } else { + printLine(`${' '.repeat(depth)}${spec ?? tree.name} - ${tree.resolved!.version}`) + for (const [k, v] of Object.entries(tree.subtrees)) { + printTree(v, depth + 1, k) + } + } +} + +function isRootTree(tree: Tree | RootTree): tree is RootTree { + return !tree.parent +} + +function findTree(tree: Tree | RootTree, spec: string) { + const match = tree.subtrees[spec] + if (match) return match + if (tree.parent) return findTree(tree.parent, spec) +} + +async function validateTree(tree: Tree | RootTree, repo: PackageRepository, stack: [spec: string, parent: Tree | RootTree][] = []) { + function fail(message: string): never { + for (const [spec, parent] of stack) { + const name = isRootTree(parent) + ? '[root]' + : `${parent.name}@${parent.resolved?.version ?? parent.constraint.source}` + + printLine(`${name} -> ${spec}`) + } + + throw new Error(message) + } + + if (!isRootTree(tree)) { + if (!tree.resolved) { + fail(`Unresolved tree: ${tree.name}@${tree.constraint.source}`) + } + + if (tree.parent !== stack[stack.length - 1][1]) { + printLine(` ${(tree.parent as any).name} ${(tree.parent as any).id}`) + fail(`Bad parent ${tree.name}@${tree.constraint.source}`) + } + + async function validateDep(parentTree: Tree & { resolved: ResolvedDependency }, k: string, v: string, isPeer = false) { + const t = findTree(tree, k) + if (!t) fail(`No dependency found: ${k}@${v}`) + if (!t.resolved) fail(`Unresolved dependency: ${k}@${v}`) + + const p = await repo.resolvePattern(k, v) + if (!isCompatible(parseVersionConstraint(t.resolved.version), p.version, isPeer)) { + fail(`Matched dependency ${t.resolved.name}@${t.resolved.version} with incompatible constraint ${v} from ${parentTree.resolved.name}@${parentTree.resolved.version}`) + } + + if (isPeer && t === tree) { + fail(`Matched invalid peer dependency ${t.resolved.name}@${t.resolved.version} from ${parentTree.resolved.name}@${parentTree.resolved.version}`) + } + } + + const pkg = await repo.getPackageJson(tree.resolved.name, tree.resolved.version) + const required = getRequired(pkg, false) + if (required) { + for (const [k, v] of Object.entries(required)) { + await validateDep(tree as any, k, v) + } + } + + if (pkg.peerDependencies) { + const meta = pkg.peerDependenciesMeta ?? {} + for (const [k, v] of Object.entries(pkg.peerDependencies)) { + const isOptional = meta[k]?.optional + if (isOptional) continue + + await validateDep(tree as any, k, v, true) + } + } + } + + for (const [k, v] of Object.entries(tree.subtrees)) { + await validateTree(v, repo, [...stack, [k, tree]]) + } +} + + +export function showManifest(m: TerraformPackageManifest, maxDepth = 0, dedupe = false) { + const seen = new Set() + + function render(spec: string, id: string, depth = 0) { + if (depth > maxDepth) { + return + } + + const pkg = m.packages[id] + const key = `${spec}-${pkg.version}-${depth}` + if (seen.has(key)) { + return + } + + if (dedupe) { + seen.add(key) + } + + // TODO: show when `spec` !== `pkg.name` + getLogger().log(`${' '.repeat(depth)}${depth ? '|__ ' : ''}${spec} - ${pkg.version} `) + const deps = m.dependencies[id] + if (deps) { + for (const [k, v] of Object.entries(deps)) { + render(k, `${v.package}`, depth + 1) + } + } + } + + for (const [k, v] of Object.entries(m.roots)) { + render(k, `${v.package}`) + } +} + +export async function testResolveDeps(target: string, cmp?: string) { + const pkg = JSON.parse(await getFs().readFile(target, 'utf-8')) + const previousInstallation = await getInstallation(getProgramFs()) + const repo = createSprRepoWrapper(getFs(), getBuildTargetOrThrow().projectId, pkg.directory) + const result = await resolveDepsGreedy({ ...pkg.dependencies, ...pkg.devDependencies }, repo, { + oldData: previousInstallation?.importMap, + }) + + const manifest = await toPackageManifestFromTree(repo, result) + await repo.close() + + return manifest +} + +function getNameAndScheme(name: string) { + const parts = name.split(':') + if (parts.length === 1) { + return { name: parts[0] } + } + + const scheme = parts[0] + const actualName = parts.slice(1).join(':') + + return { + scheme, + name: actualName, + } +} + +export async function installModules(dir: string, deps: Record, target?: Partial) { + const fs = getFs() + const repo = createSprRepoWrapper(fs, getBuildTargetOrThrow().projectId, dir) + const result = await resolveDepsGreedy(deps, repo) + + const manifest = await toPackageManifestFromTree(repo, result) + const installer = createPackageInstaller({ fs, target }) + const mapping = await installer.getImportMap(manifest) + + const res = await writeToNodeModules(fs, mapping, dir, undefined, { mode: 'all', hoist: false }) + await repo.close() + + return { res, mapping } +} + +type NodeModulesInstallMode = 'none' | 'types' | 'all' // Changes what we write to `node_modules` + +interface WriteToNodeModulesOptions { + mode?: NodeModulesInstallMode + hoist?: boolean + verifyBin?: boolean + hideInternalTypes?: boolean +} + +const isProd = process.env.SYNAPSE_ENV === 'production' + +export async function installFromSnapshot( + dir: string, + dest: string, + pkgLocation: string, + snapshot: Snapshot & { store: ReadonlyBuildFs }, + shouldUpdateDependents = false +) { + const fs = getFs() + + await fs.deleteFile(dest).catch(throwIfNotFileNotFoundError) + const p: Promise[] = [] + for (const [k, v] of Object.entries(snapshot.store.index.files)) { + if (k.endsWith('.ts') || k === 'package.json') { + p.push(getDataRepository().readData(v.hash).then(d => fs.writeFile(path.resolve(dest, k), d))) + } + } + + if (!snapshot.store.index.files['package.json']) { + p.push(fs.writeFile(path.resolve(dest, 'package.json'), await fs.readFile(path.resolve(pkgLocation, 'package.json')))) + } + + await Promise.all(p) + + if (shouldUpdateDependents) { + const data = await getDependentsData() + const obj = data[pkgLocation] ??= {} + updateDependent(obj, dir, dest, snapshot.storeHash) + + await setDependentsData(data) + } +} + +export async function writeToNodeModules( + fs: Fs, + mapping: ImportMap2, + installDir: string, + oldPackages?: Record, + opt: WriteToNodeModulesOptions = {}, +): Promise<{ packages: Record; installed: string[]; removed: string[]; changed: string[] }> { + const { + hoist = false, + mode = 'types', + verifyBin = false, + hideInternalTypes = isProd, + } = opt + + const rootDir = getRootDirectory() + const typesOnly = mode === 'types' + const ensureDir = keyedMemoize((dir: string) => require('node:fs/promises').mkdir(dir, { recursive: true }).catch((e: any) => { + if ((e as any).code !== 'EEXIST') { + throw e + } + })) + + const packages: Record = {} + + //const missing = new Set() + const noop = new Set() + const needsClean = new Set() + const needsInstall: Record = {} + const removed = oldPackages ? new Set(Object.keys(oldPackages)) : undefined + + async function verifyBinDir() { + const p: Promise[] = [] + for (const [k, v] of Object.entries(packages)) { + const pkg = v.packageFile + const bin = pkg.bin + if (!bin) continue + + const tools = typeof bin === 'string' ? [[pkg.name, bin] as const] : Object.entries(bin) + const dest = path.resolve(installDir, k) + const dir = path.dirname(path.dirname(dest)) + async function checkOrInstallBin(k: string, v: string) { + const d = path.resolve(dir, 'node_modules', '.bin', k) + if (await getFs().fileExists(d)) { + return + } + + return installBin(fs, path.resolve(dest, v), k, dir) + } + + p.push(Promise.all(tools.map(([k, v]) => checkOrInstallBin(k, v).catch(e => { + getLogger().warn(`Failed to install bin "${k}" from package: ${path.relative(installDir, dest)}`, e) + })))) + } + await Promise.all(p) + } + + function visit2(m: ImportMap2, dir: string) { + const nodeModules = path.resolve(dir, 'node_modules') + + async function visitInner(k: string, v: ImportMap2[string]) { + if (v.source?.type === 'artifact') { + return + } + + const pkgInfo = v.source?.data + const dest = path.resolve(nodeModules, k) + const directory = path.relative(installDir, dest) + const oldEntry = oldPackages?.[directory] + const p = v.mapping ? visit2(v.mapping, dest) : undefined + + if (oldEntry) { + removed?.delete(directory) + } + + if (await fs.fileExists(dest)) { + if (!oldEntry || oldEntry.integrity !== pkgInfo?.resolved?.integrity) { + needsClean.add(directory) + needsInstall[directory] = { dir, spec: k, pkgLocation: v.location, info: pkgInfo } + } else { + packages[directory] = oldEntry + } + } else { + // if (oldEntry) { + // missing.add(directory) + // } + needsInstall[directory] = { dir, spec: k, pkgLocation: v.location, info: pkgInfo } + } + + return p + } + + const promises: Promise[] = [] + for (const [k, v] of Object.entries(m)) { + promises.push(visitInner(k, v)) + } + + return Promise.all(promises).then(() => {}) + } + + const rootDeps = new Set(Object.keys(mapping)) // This assumes the mapping hasn't already been hoisted + const hoisted = hoist ? hoistImportMap(mapping) : mapping + + // First step - figure out what needs to be done + if (await getFs().fileExists(path.resolve(installDir, 'node_modules'))) { + await visit2(hoisted, installDir) + } else { + function visit2(m: ImportMap2, dir: string) { + const nodeModules = path.resolve(dir, 'node_modules') + + function visitInner(k: string, v: ImportMap2[string]) { + if (v.source?.type === 'artifact') { + return + } + + const pkgInfo = v.source?.data + const dest = path.resolve(nodeModules, k) + const directory = path.relative(installDir, dest) + + if (v.mapping) { + visit2(v.mapping, dest) + } + + needsInstall[directory] = { dir, spec: k, pkgLocation: v.location, info: pkgInfo } + } + + for (const [k, v] of Object.entries(m)) { + visitInner(k, v) + } + } + visit2(hoisted, installDir) + } + + if (verifyBin) { + await verifyBinDir() + } + + // Second step - install + const promises: Record> = {} + const sorted = Object.entries(needsInstall).sort((a, b) => strcmp(a[0], b[0])) // shortest first + for (const [k, v] of sorted) { + getLogger().log(`installing ${v.info?.name}@${v.info?.version} [${k}]`) + promises[k] = install(v.dir, v.spec, v.pkgLocation, v.info).then(x => { + packages[k] = x + }) + } + + await Promise.all(Object.values(promises)) + + async function copyFile(src: string, dest: string) { + const fs2 = require('node:fs/promises') as typeof import('node:fs/promises') + + return fs2.copyFile(src, dest).catch(async e => { + throwIfNotFileNotFoundError(e) + await fs2.mkdir(path.dirname(dest), { recursive: true }) + + return fs2.copyFile(src, dest) + }) + } + + // Used when you want to preserve existing files + async function copyIntoDir(src: string, dest: string) { + const fs2 = require('node:fs/promises') as typeof import('node:fs/promises') + + const files = await getFs().readDirectory(src) + const p: Promise[] = [] + for (const f of files) { + if (f.type === 'directory') { + p.push(fastCopyDir(path.resolve(src, f.name), path.resolve(dest, f.name))) + } else { + p.push(fs2.copyFile(path.resolve(src, f.name), path.resolve(dest, f.name))) + } + } + + await Promise.all(p) + } + + async function install(dir: string, spec: string, pkgLocation: string, pkgInfo?: PackageInfo) { + const dest = path.resolve(dir, 'node_modules', spec) + const directory = path.relative(installDir, dest) + + if (pkgInfo?.type === 'synapse-tool') { + const bin = pkgInfo.bin + if (bin) { + getLogger().debug(`Installing bin from package "${directory}"`) + const tools = typeof bin === 'string' ? [[pkgInfo?.name, bin]] : Object.entries(bin) + await Promise.all(tools.map(([k, v]) => installBin(fs, path.resolve(pkgLocation, v), k, dir).catch(e => { + getLogger().warn(`Failed to install bin "${k}" from package: ${directory}`, e) + }))) + } + + return { + name: pkgInfo?.name, + version: pkgInfo?.version, + resolved: pkgInfo?.resolved?.url, + integrity: pkgInfo?.resolved?.integrity, + directory: pkgLocation, + specifier: spec, + isSynapsePackage: pkgInfo?.resolved?.isSynapsePackage, + packageFile: { name: pkgInfo?.name ?? '' }, + } + } + + if (pkgInfo?.type === 'synapse-provider') { + noop.add(directory) + + if (mode !== 'none' && pkgInfo?.name && rootDeps.has(spec)) { + const installer = getDefaultPackageInstaller() + await installer.installProviderTypes(dir, { [pkgInfo.name]: pkgLocation }) + } + + return { + name: pkgInfo?.name, + version: pkgInfo?.version, + resolved: pkgInfo?.resolved?.url, + integrity: pkgInfo?.resolved?.integrity, + directory: pkgLocation, + specifier: spec, + isSynapsePackage: pkgInfo?.resolved?.isSynapsePackage, + packageFile: { name: pkgInfo?.name ?? '' }, + } + } + + let hasSnapshot = false + if (pkgInfo?.type !== 'npm') { // TODO: add a check to `package.json` ? + const snapshot = pkgInfo?.resolved?.isSynapsePackage + ? await getSnapshot(pkgLocation) + : await tryLoadSnapshot(pkgLocation) + + hasSnapshot = !!snapshot + + if (snapshot?.moduleManifest) { + const declarations = toTypeDeclarations(rootDir, snapshot.moduleManifest, hideInternalTypes) + getLogger().debug(`Installing types from package "${directory}": ${Object.keys(declarations.packages)}`) + const installed = await installTypes(fs, dir, declarations) + for (const [k, v] of Object.entries(installed)) { + packages[path.relative(dir, k)] = v + } + } + + if (pkgInfo?.type === 'file') { + if (snapshot?.store) { + const selfPath = getSelfPath() + const selfDir = selfPath ? path.dirname(path.dirname(selfPath)) : undefined + const updateDependents = selfDir ? !pkgLocation.startsWith(selfDir) : false + await installFromSnapshot(installDir, path.resolve(dir, 'node_modules', spec), pkgLocation, snapshot as Snapshot & { store: ReadonlyBuildFs }, updateDependents) + } else { + const link = () => fs.link(pkgLocation, dest, { symbolic: true, typeHint: 'dir' }) + await link().catch(async e => { + if ((e as any).code !== 'EEXIST') { + throw e + } + + // The symlink can get corrupted. This probably happened here. Try again. + getLogger().warn(`Removing possibly broken symlink:`, dest, (e as any).code) + await fs.deleteFile(dest) + await link() + }) + } + } + } + + const oldEntry = oldPackages?.[directory]?.packageFile + const pkg = oldEntry ?? (await getPackageJson(fs, pkgLocation, false))?.data + if (!pkg) { + throw new Error(`No package.json found: "${dest}"`) + } + + function hasNodeModules() { + if (!needsClean.has(directory)) { + return false + } + + if (pkg!.bundledDependencies) { + return true + } + + return getFs().fileExists(path.resolve(dest, 'node_modules')) + } + + if (mode !== 'none' && pkgInfo?.type !== 'file') { + // We need to wait for the parent copy to finish when using fast copy + const parentDir = path.relative(installDir, dir) + await promises[parentDir] + + const _hasNodeModules = await hasNodeModules() + + if (needsClean.has(directory)) { + if (_hasNodeModules) { + await cleanDir(dest, ['node_modules']) + } else { + await removeDir(dest).catch(throwIfNotFileNotFoundError) + } + } + + if (!_hasNodeModules && (spec.startsWith('@') || dir !== installDir)) { + await ensureDir(path.dirname(dest)) + } + + try { + if (_hasNodeModules) { + await copyIntoDir(pkgLocation, dest) + } else { + await fastCopyDir(pkgLocation, dest) + } + } catch (e) { + if ((e as any).code !== 'EEXIST') { + if (!(e as any).message.includes('is not available in the current runtime')) { + getLogger().debug(`Fast copy failed on "${directory}"`, e) + } + + // Copy `package.json` to help TypeScript + const patterns = typesOnly ? ['**/*.d.ts', 'package.json'] : ['*'] + // A decent chunk of a time is just spent globbing + // We also aren't streaming the discovered files + const files = await glob(fs, pkgLocation, patterns) + await Promise.all(files.map(async f => copyFile(f, path.resolve(dest, path.relative(pkgLocation, f))))) + } + } + } + + const bin = pkg.bin + if (bin && mode !== 'none') { + getLogger().debug(`Installing bin from package "${path.relative(installDir, dest)}"`) + const tools = typeof bin === 'string' ? [[pkg.name, bin]] : Object.entries(bin) + await Promise.all(tools.map(([k, v]) => installBin(fs, path.resolve(dest, v), k, dir).catch(e => { + getLogger().warn(`Failed to install bin "${k}" from package: ${path.relative(installDir, dest)}`, e) + }))) + } + + return { + name: pkg.name, + version: pkgInfo?.version, + resolved: pkgInfo?.resolved?.url, + integrity: pkgInfo?.resolved?.integrity, + directory: pkgInfo?.type !== 'file' ? directory : pkgLocation, + packageFile: oldEntry ?? prunePkgJson(pkg), + specifier: spec, + isSynapsePackage: pkgInfo?.resolved?.isSynapsePackage, + hasSnapshot, + } + } + + return { + packages, + installed: Object.keys(needsInstall).filter(x => !needsClean.has(x) && !noop.has(x)), + removed: removed ? [...removed] : [], + changed: [...needsClean].filter(x => x in packages), + } +} + +async function toPackageManifestFromTree(repo: PackageRepository, tree: { subtrees: Record }, oldData?: ReturnType): Promise { + const roots: Record = {} + const packages = new Map() + const dependencies: Record> = {} + + function getPackageId(info: PackageInfo) { + if (!packages.has(info)) { + packages.set(info, packages.size) + } + + return packages.get(info)! + } + + const oldPackages = new Map() + if (oldData) { + for (const v of Object.values(oldData.sources)) { + if (v?.type !== 'package') continue + const key = `${v.data.type}-${v.data.name}-${v.data.version}` + if (!oldPackages.has(key)) { + oldPackages.set(key, v.data) + } + } + } + + const packageInfos = new Map() + async function getPackageInfo(gid: number, resolved: ResolvedDependency) { + if (packageInfos.has(gid)) { + return packageInfos.get(gid)! + } + + const isProvider = resolved.name.startsWith(providerPrefix) + const isTool = resolved.name.startsWith(toolPrefix) + const parsed = getNameAndScheme(resolved.name) + const type: PackageInfo['type'] = parsed.scheme as any ?? 'npm' + const key = `${type}-${resolved.name}-${resolved.version}` + if (oldPackages.has(key)) { + const info = oldPackages.get(key)! + const copy = { ...info } + packageInfos.set(gid, copy) + + return copy + } + + const pkgJson = await repo.getPackageJson(resolved.name, resolved.version) + + const isSynapsePackage = !!pkgJson.synapse || pkgJson.dist.isSynapsePackage + const info: PackageInfo = { + type: type === 'npm' && pkgJson?.synapse ? 'spr' : type, + name: (isProvider || isTool) ? pkgJson.name : parsed.name, + version: resolved.version, + os: pkgJson.os, + cpu: pkgJson.cpu, + bin: pkgJson.bin, + resolved: { + integrity: pkgJson.dist.integrity, + url: pkgJson.dist.tarball, + isStubPackage: pkgJson.dist.isStubPackage, + isSynapsePackage, + }, + } + + Object.assign(info, { + dependencies: resolved.dependencies, + peers: resolved.peers, + }) + + packageInfos.set(gid, info) + + return info + } + + const constraints: Record = {} + const trees: Record = {} + + async function visit(tree: Tree) { + if (!tree.resolved) { + throw new Error(`Found unresolved tree: ${tree.name}`) + } + const info = await getPackageInfo(tree.id, tree.resolved) + const id = getPackageId(info) + constraints[tree.id] = { package: id, versionConstraint: tree.constraint.source! } + trees[tree.id] = tree + + for (const [k, v] of Object.entries(tree.subtrees)) { + await visit(v) + } + } + + for (const [k, v] of Object.entries(tree.subtrees)) { + await visit(v) + } + + for (const [k, v] of Object.entries(trees)) { + const id = constraints[v.id].package + const deps = dependencies[id] ??= {} + for (const [k2, v2] of Object.entries(v.subtrees)) { + deps[k2] = constraints[v2.id] + } + } + + for (const [spec, t] of Object.entries(tree.subtrees)) { + roots[spec] = constraints[t.id] + } + + return { + roots, + dependencies, + packages: Object.fromEntries([...packages.entries()].map(i => i.reverse() as [number, PackageInfo])), + } +} + +async function toPackageManifest(repo: PackageRepository, resolved: ResolveDepsResult): Promise { + const roots: Record = {} + const packages = new Map() + const dependencies: Record> = {} + + function getPackageId(info: PackageInfo) { + if (!packages.has(info)) { + packages.set(info, packages.size) + } + + return packages.get(info)! + } + + const packageInfos = new Map() + async function getPackageInfo(gid: number, resolved: ResolvedDependency) { + if (packageInfos.has(gid)) { + return packageInfos.get(gid)! + } + + const pkgJson = await repo.getPackageJson(resolved.name, resolved.version) + + const isProvider = resolved.name.startsWith(providerPrefix) + const isTool = resolved.name.startsWith(toolPrefix) + + const isSynapsePackage = !!pkgJson.synapse || pkgJson.dist.isSynapsePackage + const parsed = getNameAndScheme(resolved.name) + const type: PackageInfo['type'] = parsed.scheme as any ?? 'npm' + const info: PackageInfo = { + type, + name: (isProvider || isTool) ? pkgJson.name : parsed.name, + version: resolved.version, + os: pkgJson.os, + cpu: pkgJson.cpu, + bin: pkgJson.bin, + resolved: { + integrity: pkgJson.dist.integrity, + url: pkgJson.dist.tarball, + isStubPackage: pkgJson.dist.isStubPackage, + isSynapsePackage, + }, + } + packageInfos.set(gid, info) + + return info + } + + const constraints: Record = {} + for (const [k, v] of resolved.installed) { + const info = await getPackageInfo(k.id, v) + const id = getPackageId(info) + constraints[k.id] = { package: id, versionConstraint: k.constraint.source! } + } + + for (const [k, v] of resolved.installed) { + const id = constraints[k.id].package + for (const [k2, v2] of Object.entries(v.dependencies)) { + const deps = dependencies[id] ??= {} + deps[k2] = constraints[v2] + } + + // TODO: peer deps at the root require creating a new root + // Peers are added to all parent constraint groups + for (const [k2, v2] of Object.entries(v.peers)) { + // This is a peer dependency of a root dep, add it as a root dep + if (!k.parents) { + roots[k2] = constraints[v2] + continue + } + + for (const parentId of k.parents.map(p => p.id)) { + const id = constraints[parentId].package + const deps = dependencies[id] ??= {} + deps[k2] = constraints[v2] + } + } + } + + for (const [spec, id] of Object.entries(resolved.roots)) { + roots[spec] = constraints[id] + } + + return { + roots, + dependencies, + packages: Object.fromEntries([...packages.entries()].map(i => i.reverse() as [number, PackageInfo])), + } +} + + +export async function downloadDependency(dir: string, dep: PublishedPackageJson) { + const client = createNpmRegistryClient() + const key = `${dep.name}-${dep.version}` + const dest = path.resolve(dir, key) + if (await fs.access(dest, fs.constants.F_OK).then(() => true, () => false)) { + getLogger().log(`Using cached package: ${key}`) + + return dest + } + + const tarball = await client.downloadPackage(dep.dist.tarball) + const files = extractTarball(tarball) + for (const f of files) { + const absPath = path.resolve(dest, f.path.replace(/^package\//, '')) + await fs.mkdir(path.dirname(absPath), { recursive: true }) + await fs.writeFile(absPath, f.contents, { mode: f.mode }) + } + + return dest +} + +export async function listInstall(dir = getWorkingDir(), programFs: Pick = getProgramFs()) { + const installation = await getInstallation(programFs) + const pkgs = installation?.packages + if (!pkgs) { + return + } + + interface Pkg { + readonly name: string + readonly version: string + readonly deps: Record + readonly isRoot?: boolean + } + + const trees: Record = {} + function getTree(dir: string[]): Pkg { + const p = dir.flatMap(x => ['node_modules', x]).join(path.sep) + if (dir.length === 1) { + return trees[dir[0]] ??= { + name: pkgs![p].name, + version: pkgs![p].version!, + deps: {}, + isRoot: true, + } + } + + const t = getTree(dir.slice(0, -1)) + return t.deps[dir.at(-1)!] ??= { + name: pkgs![p].name, + version: pkgs![p].version!, + deps: {}, + } + } + + function printTree(pkg: Pkg, depth = 0) { + printLine(' '.repeat(depth) + `${pkg.name}@${pkg.version}`) + for (const p of Object.values(pkg.deps)) { + printTree(p, depth + 1) + } + } + + const dirs = Object.keys(pkgs).map(x => x.split('node_modules').map(x => { + if (x.endsWith(path.sep)) { + x = x.slice(0, -1) + } + if (x.startsWith(path.sep)) { + x = x.slice(1) + } + return x + }).filter(x => !!x)).filter(x => x.length > 0).sort((a, b) => { + const d = a.length - b.length + if (d !== 0) { + return d + } + + return strcmp(a[a.length - 1], b[b.length - 1]) + }) + + for (const d of dirs) { + getTree(d) + } + + for (const t of Object.values(trees)) { + if (t.isRoot) { + printTree(t) + } + } +} + +export function importMapToManifest(importMap: NonNullable) { + const manifest: TerraformPackageManifest = { roots: {}, dependencies: {}, packages: {} } + const root = importMap.mappings['#root'] + for (const [k, v] of Object.entries(root)) { + const source = importMap.sources[v] + if (source?.type !== 'package') continue + + manifest.roots[k] = { package: Number(v), versionConstraint: source.data.version } + } + + for (const [k, v] of Object.entries(importMap.sources)) { + if (v?.type !== 'package') continue + manifest.packages[k] = v.data + } + + for (const [k, v] of Object.entries(importMap.mappings)) { + if (v === root) continue + + manifest.dependencies[k] = Object.fromEntries(Object.entries(v).map(([x, y]) => { + const source = importMap!.sources[y] + if (source?.type !== 'package') { throw new Error(`not implemented: ${source?.type}`) } + + return [x, { package: Number(y), versionConstraint: source.data.version }] + })) + } + + return manifest +} + +export async function verifyInstall(dir = getWorkingDir(), programFs: Pick = getProgramFs()) { + const installation = await getInstallation(programFs) + if (!installation?.importMap) { + return + } + + const manifest = importMapToManifest(installation.importMap) + + const view = createInstallView() + getLogger().emitInstallEvent({ phase: 'start' }) + + const installer = getDefaultPackageInstaller() + const mapping = await installer.getImportMap(manifest) + + getLogger().emitInstallEvent({ phase: 'write' }) + const mode = installation.mode + const res = await writeToNodeModules(getFs(), mapping, dir, installation.packages, { mode, verifyBin: true }) + await commitPackages(programFs, res.packages, flattenImportMap(mapping, false), Date.now(), mode) + + const pkg = await getCurrentPkg() + const deps = (pkg ? getRequired(pkg.data, undefined, true) : undefined) ?? {} + + const summary = await createSummary(deps, mapping, res, installation.importMap) + + getLogger().emitInstallEvent({ phase: 'end', summary } as any) + view.summarize() + + await getNpmPackageRepo().close() +} + + +export async function downloadAndInstall( + pkg: ResolvedPackage, + service: PackageInstaller = getDefaultPackageInstaller(), + installAll = false +) { + const fs = getFs() + const programFs = getProgramFs() + const previousInstallation = await getInstallation(programFs) + + const csDeps = pkg?.data.synapse?.dependencies ?? {} + const providers = pkg?.data.synapse?.providers ?? {} + const tools = pkg?.data.synapse?.devTools ?? {} + + const deps: Record = { + ...pkg.data.dependencies, + ...pkg.data.devDependencies, + ...Object.fromEntries(Object.entries(csDeps).map(x => [x[0], `spr:${x[1]}`])), + ...Object.fromEntries(Object.entries(providers).map(x => createSynapseProviderRequirement(x[0], x[1]))), + ...Object.fromEntries(Object.entries(tools).map(x => [`${toolPrefix}${x[0]}`, x[1]])), + ...resolveWorkspaces(pkg.data.workspaces ?? [], pkg.directory) + } + + getLogger().emitInstallEvent({ phase: 'start' }) + + const repo = createSprRepoWrapper(fs, getBuildTargetOrThrow().projectId, pkg.directory) + const doResolve = () => resolveDepsGreedy(deps, repo, { oldData: previousInstallation?.importMap, validateResult: false }) + const result = await runTask('install', 'resolve deps', doResolve, 10) + + //getLogger().log(`Total specifiers resolved: ${result.installed.length}`) + const oldPackages = previousInstallation?.packages + + const createManifest = () => toPackageManifestFromTree(repo, result, previousInstallation?.importMap) + + const manifest = await runTask('install', 'create manifest', createManifest, 10) + const getImportMap = () => service.getImportMap(manifest) + + const mapping = await runTask('install', 'get import map', getImportMap, 10) + + getLogger().emitInstallEvent({ phase: 'write' }) + + const mode = 'all' + const doWrite = () => writeToNodeModules(getFs(), mapping, pkg.directory, oldPackages, { + mode, + hoist: false, + hideInternalTypes: pkg.data.name === 'synapse' ? false : undefined, + }) + + const res = await runTask('install', 'write node_modules', doWrite, 100) + + await commitPackages(programFs, res.packages, flattenImportMap(mapping, false), Date.now(), mode) + + getLogger().log('Creating summary') + const summary = await createSummary(deps, mapping, res, previousInstallation?.importMap, repo) + + await repo.close() + + getLogger().emitInstallEvent({ phase: 'end', summary } as any) + + return result +} + +async function createSummary( + deps: Record, + mapping: ImportMap2, + writeResult: Awaited>, + oldMapping?: ReturnType, + repo: PackageRepository = getNpmPackageRepo() +): Promise { + const current: Record = {} + const summary: InstallSummary = { rootDeps: {}, ...writeResult } + + const maybeNeedsLatestVersion = new Set() + + for (const k of Object.keys(deps)) { + const m = mapping[k] + const s = m?.source + if (!s || s.type !== 'package') continue + + if (s.data.type === 'npm') { + maybeNeedsLatestVersion.add(k) + } + + current[k] = s.data.version + summary.rootDeps[k] = { + name: s.data.name, + installedVersion: s.data.version, + } + } + + async function getDepsForComparison() { + const oldPkg = await getProgramFs().readFile('package.json', 'utf-8') + .then(JSON.parse).catch(throwIfNotFileNotFoundError) + + if (!oldPkg) { + return + } + + return { ...oldPkg.dependencies, ...oldPkg.devDependencies } + } + + const previousDeps: Record = {} + const root = oldMapping?.mappings['#root'] + if (root) { + for (const [k, v] of Object.entries(await getDepsForComparison())) { + const source = oldMapping.sources[root[k]] + if (source?.type !== 'package') continue + + previousDeps[k] = source.data.version + if (!(k in summary.rootDeps)) { + summary.rootDeps[k] = { + name: source.data.name, + installedVersion: source.data.version, + } + } + } + } + + const diff = diffPkgDeps({ dependencies: current }, { dependencies: previousDeps }) + + function shouldFetchLatestVersion(k: string) { + return diff?.added?.[k] || diff?.changed?.[k] + } + + for (const k of maybeNeedsLatestVersion) { + if (shouldFetchLatestVersion(k)) { + const latestVersion = (await repo.resolvePattern(k, 'latest')).version + summary.rootDeps[k].latestVersion = latestVersion.source + } + } + + return { ...summary, diff } +} + + +// TODO: lifecycle hooks +// https://github.com/oven-sh/bun/issues/9527 + +export function getDefaultPackageInstaller(fs = getFs()) { + return createPackageInstaller({ + fs, + getProviderGenerator: memoize(async () => { + const tfPath = await getTerraformPath() + + return createProviderGenerator(fs, providerRegistryHostname, tfPath) + }) + }) +} + +export async function maybeDownloadPackages(pkg: ResolvedPackage, installAll?: boolean, alwaysRun?: boolean) { + const fs = getProgramFs() + const doInstall = () => downloadAndInstall(pkg, getDefaultPackageInstaller(), installAll).then(res => { + installations.delete(fs) + return res + }) + + if (alwaysRun) { + return doInstall() + } + + const hasInstallation = await getInstallationCached(fs) + if (!hasInstallation) { + return doInstall() + } + + await runIfPkgChanged(doInstall) +} + +export async function downloadAndUpdatePackage(pkg: ResolvedPackage, deps: Record, isDev?: boolean, installAll?: boolean, isRemove?: boolean) { + const d = pkg.data as Mutable + const currentDeps = isDev ? d.devDependencies ??= {} : d.dependencies ??= {} + for (const [k, v] of Object.entries(deps)) { + if (isRemove) { + if (d.dependencies) { + delete d.dependencies[k] + } + if (d.devDependencies) { + delete d.devDependencies[k] + } + } else { + currentDeps[k] = v + if (isDev && d.dependencies && k in d.dependencies) { + delete d.dependencies[k] + } else if (!isDev && d.devDependencies && k in d.devDependencies) { + delete d.devDependencies[k] + } + } + } + + const installer = getDefaultPackageInstaller() + const result = await downloadAndInstall(pkg, installer, installAll) + + for (const [k, v] of Object.entries(result.subtrees)) { + if (isRemove || !(k in deps)) continue + const z = parseVersionConstraint(deps[k]) + if (isExact(z)) { + currentDeps[k] = `${v.resolved!.version}` + } else { + currentDeps[k] = `^${v.resolved!.version}` + } + } + + await setCompiledPkgJson(pkg.data) + await commitProgram() + + // We don't use `pkg.data` directly because it's the "compiled" (transformed) package.json + const userPkg: Omit = await getPackageJson(getFs(), pkg.directory, false).then(r => r?.data) ?? {} + for (const k of Object.keys(deps)) { + const dep = pkg.data.dependencies?.[k] + if (dep) { + const userDeps = (userPkg as Mutable).dependencies ??= {} + userDeps[k] = dep + } else { + if ((userPkg as Mutable).dependencies) { + delete (userPkg as Mutable).dependencies![k] + } + } + const devDep = pkg.data.devDependencies?.[k] + if (devDep) { + const userDevDeps = (userPkg as Mutable).devDependencies ??= {} + userDevDeps[k] = devDep + } else { + if ((userPkg as Mutable).devDependencies) { + delete (userPkg as Mutable).devDependencies![k] + } + } + } + + await getFs().writeFile( + path.resolve(pkg.directory, 'package.json'), + JSON.stringify(userPkg, undefined, 4) + ) +} + + +interface PackageTypeDeclarations { + files: Record + roots: string[] + symbols?: TypesFileData + sourcemaps?: Record +} + +interface TypeDeclarations { + readonly packages: Record +} + +// `rootDir` is used for sourcemaps +function toTypeDeclarations(rootDir: string, manifest: Record>, hideInternal?: boolean) { + const packages: Record = {} + for (const [k, v] of Object.entries(manifest)) { + if (!v.types) { + continue + } + + if (hideInternal && v.internal) { + continue + } + + const namespace = k.split(':')[0] + const declarations: PackageTypeDeclarations = packages[namespace] ??= { files: {}, roots: [] } + declarations.files[v.types.name] = v.types.text + declarations.roots.push(v.types.name) + + if (v.types.sourcemap && !v.types.sourcemap.startsWith('pointer:')) { + const maps = declarations.sourcemaps ??= {} + const mapping = JSON.parse(v.types.sourcemap) + mapping.sourceRoot = rootDir + maps[`${v.types.name}.map`] = JSON.stringify(mapping) + } + + if (v.types.symbols) { + const symbols = declarations.symbols ??= {} + symbols[v.types.name] = v.types.symbols + } + + // Not implemented + // if (v.types.references) { + // for (const [k2, v2] of Object.entries(v.types.references)) { + // declarations.files[k2] = getFs().readFileSync(path.resolve(pkgDir, v2), 'utf-8') + // } + // } + } + + return { packages } +} + +async function installTypePackage(fs: Fs, typesDir: string, name: string, types: PackageTypeDeclarations) { + const dest = path.resolve(typesDir, name) + + async function writeAll(files: Record) { + await Promise.all(Object.entries(files).map(([k, v]) => fs.writeFile(path.resolve(dest, k), v))) + } + + await writeAll(types.files) + if (types.sourcemaps) { + await writeAll(types.sourcemaps) + } + + const indexText = types.roots.sort() + .map(p => `/// `) + .join('\n') + + const indexDest = path.resolve(dest, 'index.d.ts') + await fs.writeFile(indexDest, indexText) + + const pkgJson = { + name: `@types/${name}`, + types: 'index.d.ts', + } + + await fs.writeFile( + path.resolve(dest, 'package.json'), + JSON.stringify(pkgJson, undefined, 4) + ) + + return [dest, { + name: pkgJson.name, + directory: dest, + packageFile: pkgJson, + types: types.symbols, + }] satisfies [string, NpmPackageInfo] +} + +async function installTypes(fs: Fs, rootDir: string, types: TypeDeclarations): Promise> { + const packages: Record = {} + const typesDir = path.resolve(rootDir, 'node_modules', '@types') + for (const [k, v] of Object.entries(types.packages)) { + const [dir, info] = await installTypePackage(fs, typesDir, k, v) + packages[dir] = info + } + + return packages +} + +function checkOs(info: Pick, platform = process.platform) { + if (!info.os) { + return + } + + let isNegation = false + for (const os of info.os) { + if (os.startsWith('!')) { + isNegation = true + if (platform === os.slice(1)) { + return false + } + } else if (platform === os) { + return true + } + } + + return isNegation +} + +function checkCpu(info: Pick, arch = process.arch) { + if (!info.cpu) { + return + } + + let isNegation = false + for (const cpu of info.cpu) { + if (cpu.startsWith('!')) { + isNegation = true + if (arch === cpu.slice(1)) { + return false + } + } else if (arch === cpu) { + return true + } + } + + return isNegation +} + +async function createIndexFromLockFile( + fs: Fs, + packagesDir: string, + workingDirectory: string, + packageLock: Awaited> +) { + const packages: Record = {} + const indexed = new Set() + const pendingIndexes = new Map>() + + async function indexPackageLock(packageLock: Awaited>) { + if (indexed.has(packageLock.directory)) { + return + } + + indexed.add(packageLock.directory) + + const entries = await Promise.all(Object.keys(packageLock.data.packages).map(indexEntry)) + for (const [k, v] of entries.filter(isNonNullable)) { + packages[k] = v + } + + const copied = new Map>() + await Promise.all(entries.filter(isNonNullable).map(([k, v]) => copyPackage(k, v))) + + // Just so we don't redownload things we've already seen + function copyPackage(absPath: string, info: NpmPackageInfo) { + // Only copy packages from a registry + if (!info.resolved || !info.integrity || !info.version) { + return + } + + const qualifiedName = `${info.name}-${info.version}` + const dest = path.resolve(packagesDir, 'npm', qualifiedName) + if (absPath === dest) { + return + } + + if (copied.has(dest)) { + return copied.get(dest) + } + + const packagePath = path.resolve(dest, 'package.json') + + copied.set(dest, (async function () { + if (await fs.fileExists(packagePath)) { + return + } + + // TODO: delete anything that exists here? + throw new Error(`cp not implemented for fs`) + // await fs.cp(absPath, dest, { recursive: true }) + })()) + } + + function indexEntry(key: string): Promise<[string, NpmPackageInfo] | undefined> { + const value = packageLock.data.packages[key] + if (!value) { + throw new Error(`Package lock entry not found: ${key}`) + } + + const absPath = path.resolve(packageLock.directory, key) + if (pendingIndexes.has(absPath)) { + return pendingIndexes.get(absPath)! + } + + const p = inner() + pendingIndexes.set(absPath, p) + + return p + + async function inner(): Promise<[string, NpmPackageInfo] | undefined> { + if (value.link) { + if (value.resolved === undefined) { + throw new Error(`Found unresolved linked package: ${key}`) + } + + const entry = await indexEntry(value.resolved) + if (!entry) { + // throw new Error(`Missing package.json file: ${key}`) + return + } + + const lockFile = await getNearestPackageLock(fs, entry[0]) + if (lockFile.directory !== packageLock.directory) { + await indexPackageLock(lockFile) + } + + return [absPath, entry[1]] + } + + const packageJson = await getPackageJson(fs, absPath, false) + if (!packageJson) { + // throw new Error(`Missing package.json file: ${key}`) + return + } + + return [absPath, { + name: packageJson.data.name, + directory: packageJson.directory, + packageFile: packageJson.data, + resolved: value.resolved, + integrity: value.integrity, + version: value.version ?? packageJson.data.version, + }] + } + } + } + + await indexPackageLock(packageLock) + + return Object.fromEntries(Object.entries(packages).map(([k, v]) => [path.relative(workingDirectory, k), v] as const)) +} + +export type PackageResolver = ReturnType +function createPackageResolver( + fs: Fs & SyncFs, + workingDirectory: string, + moduleResolver: ModuleResolver, + packages: Record, + packageIds?: Map +) { + + function findEntryFromPath(absPath: string): NpmPackageInfo | undefined { + const entry = packages[absPath] + if (entry) { + return entry + } + + const parentDir = path.dirname(absPath) + if (parentDir !== absPath) { + return findEntryFromPath(parentDir) + } + } + + function findEntryFromSpecifier(name: string, prefix: string): NpmPackageInfo | undefined { + const suffix = path.join('node_modules', name) + const target = path.resolve(prefix, suffix) + const entry = packages[target] + if (entry) { + return entry + } + + const parentDir = path.dirname(prefix) + if (parentDir !== prefix) { + return findEntryFromSpecifier(name, parentDir) + } + } + + + function resolveExport(packageJson: { directory: string; data: Pick }, absPath: string) { + const p = path.extname(absPath) === '' ? `${absPath}.js` : absPath + const exports = packageJson?.data.exports + + if (typeof exports !== 'object' || !exports) { + throw new Error(`Package is missing exports: ${packageJson?.directory}`) + } + + const p2 = fs.fileExistsSync(path.resolve(absPath, 'index.js')) + ? path.resolve(absPath, 'index.js') + : p + + const inverted = Object.fromEntries(Object.entries(exports).map(x => x.reverse()).map(x => [path.resolve(packageJson.directory, x[0] as any), x[1]])) + const res = inverted[p2] + if (!res) { + const mainPath = packageJson.data.main ? path.resolve(packageJson.directory, packageJson.data.main) : undefined + if (mainPath === p2) { + getLogger().log(`Using "main" export for package: ${packageJson.directory}`) + + return packageJson.data.name + } + + throw new Error(`No module export found for module "${absPath}" in package ${packageJson.data.name}`) + } + + return path.posix.join(packageJson.data.name, res as any) + } + + const mapCache = new Map() // Used to maintain reference equality + function mapEntry(entry: NpmPackageInfo): PackageInfo | undefined { + if (!entry.version) { + return + } + + if (mapCache.has(entry)) { + return mapCache.get(entry) + } + + const type: PackageInfo['type'] = entry.packageFile.synapse ? 'spr' : 'npm' + + const result = { + type, + name: entry.name, + version: entry.version, + resolved: entry.resolved ? { + url: entry.resolved, + integrity: entry.integrity, + } : undefined, + } + + mapCache.set(entry, result) + + return result + } + + const lookupCache = new Map() // Used to maintain reference equality + function reverseLookup(specifier: string, location?: string, virtualId?: string): LookupResult { + const key = `${location}::${specifier}` + if (lookupCache.has(key)) { + return lookupCache.get(key)! + } + + if (virtualId) { + const sourceNode = moduleResolver.getSource(virtualId) + const source = sourceNode?.source + if (source && packageIds?.has(source)) { + if (isRelativeSpecifier(specifier) && source.type === 'package') { + if (sourceNode.subpath) { + return { + module: `${source.data.name}/${sourceNode.subpath}`, + packageId: packageIds.get(source)!, + } + } else { + // Falls through + } + } else { + return { + module: specifier, + packageId: packageIds.get(source)!, + } + } + } + } + + const result = worker() + lookupCache.set(key, result) + + return result + + function worker() { + const lookupDir = location !== undefined + ? path.dirname(path.resolve(workingDirectory, location)) + : workingDirectory + + if (specifier.startsWith(providerPrefix)) { + const info = moduleResolver.resolveProvider(specifier, lookupDir) + const packageInfo: PackageInfo = { + type: 'synapse-provider', + name: info.name, + version: info.version, + resolved: { + url: info.source, + } + } + + return { + module: specifier, + packageInfo, + location: info.location, + } + } + + if (isRelativeSpecifier(specifier) || path.isAbsolute(specifier)) { + const resolved = path.resolve(lookupDir, specifier) + const inversedGlobals = moduleResolver.getInverseGlobalsMap() + if (inversedGlobals[resolved]) { + return { module: inversedGlobals[resolved] } + } + + if (path.extname(resolved) === '' && inversedGlobals[resolved + '.js']) { + return { module: inversedGlobals[resolved + '.js'] } + } + + const packageEntry = findEntryFromPath(resolved) + if (packageEntry === undefined) { + throw new Error(`Missing package entry: ${resolved} [resolving ${specifier} from ${location}]`) + } + + const module = resolveExport({ directory: packageEntry.directory, data: packageEntry.packageFile }, resolved) + const packageInfo = mapEntry(packageEntry) + if (!packageInfo) { + return { module } + } + + return { + module, + packageInfo, + // dependencies: getDependencies(packageEntry), + // location: packageEntry.directory, + } + } + + // Bare specifier + const components = getSpecifierComponents(specifier) + const name = components.name + const packageEntry = findEntryFromSpecifier(name, lookupDir) + if (packageEntry === undefined) { + throw new Error(`Missing package entry: ${name}`) + } + + const packageInfo = mapEntry(packageEntry) + if (!packageInfo) { + return { module: specifier } + } + + return { + module: specifier, + // packageInfo, + // dependencies: getDependencies(packageEntry), + // location: packageEntry.directory, + } + } + } + + return { reverseLookup } +} + +function getPackageDest(packagesDir: string, info: PackageInfo) { + if (info.type === 'file') { + const location = info.resolved?.url + if (!location) { + throw new Error(`Found file package without a resolved file path: ${info.name}`) + } + + return location + } + + if (info.type === 'synapse-provider') { + const source = info.resolved!.url + + return path.resolve(packagesDir, info.type, source, `${info.name}-${info.version}`) + } + + const type = info.type ?? 'npm' + const qualifiedName = type === 'cspm' || type === 'spr' + ? `${info.name}-${info.resolved!.integrity!.slice(0, 10)}` + : `${info.name}-${info.version}` + + return path.resolve(packagesDir, type, qualifiedName) +} + +async function _getProviderGenerator() { + const tfPath = await getTerraformPath() + + return createProviderGenerator(getFs(), providerRegistryHostname, tfPath) +} + +interface PackageInstallerParams { + readonly fs?: Fs & SyncFs + readonly packagesDir?: string + getProviderGenerator?: () => Promise> + target?: Partial +} + +type PackageInstaller = ReturnType + +export function createPackageInstaller(params?: PackageInstallerParams) { + const { + target, + fs = getFs(), + getProviderGenerator = memoize(_getProviderGenerator), + packagesDir = getPackageCacheDirectory(), + } = params ?? {} + + const resolvedTarget = resolveBuildTarget(target) + const nodePlatform = toNodePlatform(resolvedTarget.os) + const nodeArch = toNodeArch(resolvedTarget.arch) + + const repo = getDataRepository(fs) + const packageDownloads = new Map>() + const mappingCacheDir = path.resolve(getGlobalCacheDirectory(), 'import-maps') + + function ensurePackage(info: PackageInfo) { + if (info.resolved?.isStubPackage) { + return { type: 'skipped', reason: 'stub package' } as PackageInstallResult + } + + const type = info.type ?? 'npm' + const url = info.resolved?.url + if (!url) { + return { type: 'err', reason: new Error(`Missing download url`) } as const + } + + if (checkOs(info, nodePlatform) === false || checkCpu(info, nodeArch) === false) { + // getLogger().debug(`Skipping package "${info.name}" due to os or cpu requirements`) + getLogger().emitPackageProgressEvent({ phase: 'download', package: url, done: true, skipped: true }) + + return { type: 'skipped', reason: 'incompatible cpu or os' } as PackageInstallResult + } + + const dest = getPackageDest(packagesDir, info) + if (packageDownloads.has(dest)) { + return packageDownloads.get(dest)! + } + + const pending = (async function (): Promise { + try { + const res = await maybeDownload() + getLogger().emitPackageProgressEvent({ phase: 'download', package: url, done: true, cached: res.cacheHit }) + + return { type: 'ok', name: info.name, destination: res.dest, cacheHit: res.cacheHit } + } catch (e) { + return { type: 'err', reason: e as Error } + } + })() + + packageDownloads.set(dest, pending) + + return pending + + async function maybeDownload() { + // const packagePath = path.resolve(dest, 'package.json') + const prefetched = info.type === 'npm' && getNpmPackageRepo().isDownloading(dest) + if (await fs.fileExists(dest) && !prefetched) { + return { cacheHit: true, dest } + } + + if (type !== 'file' && !prefetched) { + getLogger().debug(`Downloading package "${info.name}-${info.version}"`) + } + + switch (type) { + case 'jsr': + case 'npm': + await downloadNpmPackage(info, dest) + break + case 'github': + await downloadGitHubPackage(info, dest) + break + case 'spr': + case 'cspm': + await downloadSynapsePackage(info, dest) + break + case 'synapse-provider': + await downloadSynapseProvider(info, dest) + break + case 'synapse-tool': + await downloadToolPackage(info, dest) + break + case 'file': + break + } + + return { cacheHit: false, dest } + } + } + + async function downloadSynapsePackage(info: PackageInfo, dest = getPackageDest(packagesDir, info)) { + const resp = await packages.client.getPackageData(info.resolved!.url, info.version) + const data = Buffer.from(resp.data, 'base64') + const dataHash = createHash('sha512').update(data).digest('base64url') + if (dataHash !== info.resolved!.integrity) { + throw new Error(`Integrity check failed for package "${info.name}"`) // FIXME: could have a better error + } + + const files = extractTarball(await gunzip(data)) + if (isSnapshotTarball(files)) { + getLogger().log('Unpacking snapshot...') + await unpackSnapshotTarball(repo, files, dest) + } else { + await Promise.all(files.map(async f => { + const absPath = path.resolve(dest, f.path) + await fs.writeFile(absPath, f.contents) + })) + } + + await fs.writeFile( + path.resolve(dest, 'package.json'), + JSON.stringify(resp.packageFile, undefined, 4) + ) + + return dest + } + + async function downloadToolPackage(info: PackageInfo, dest = getPackageDest(packagesDir, info)) { + const url = info.resolved!.url! + const data = await fetchData(url) + await extractToDir(data, dest, path.extname(url) as any) + + return dest + } + + async function downloadSynapseTool(name: string, version?: string) { + const tool = await packages.client.getTool(name) + + // FIXME + const latest = Object.values(tool.versions).pop() + if (!latest) { + throw new Error(`Tool has no published versions`) + } + + const resp = await packages.client.getToolData(latest[0].hash) + const data = Buffer.from(resp.data, 'base64') + const dataHash = createHash('sha512').update(data).digest('base64url') + if (dataHash !== latest[0].packageDataHash) { + throw new Error(`Integrity check failed for package "${name}"`) // FIXME: could have a better error + } + + + const resolvedVersion = latest[0].version + const dest = path.resolve(getToolsDirectory(), `${name}-${latest[0].hash.slice(0, 16)}`) + + const files = extractTarball(await gunzip(data)) + await Promise.all(files.map(async f => { + const absPath = path.resolve(dest, f.path) + await fs.writeFile(absPath, f.contents) + })) + + await fs.writeFile( + path.resolve(dest, 'package.json'), + JSON.stringify(latest[0].packageFile, undefined, 4) + ) + + return { + hash: latest[0].hash, + version: resolvedVersion, + destination: dest, + entrypoint: latest[0].entrypoint, + } + } + + async function downloadSynapseProvider(info: PackageInfo, dest = getPackageDest(packagesDir, info)) { + const name = info.name + const source = info.resolved!.url + const version = info.version + + const generator = await getProviderGenerator() + await generator.generate({ name, source, version }, dest) + + return dest + } + + + async function downloadNpmPackage(info: PackageInfo, dest: string) { + await getNpmPackageRepo().maybeDownloadPackage(info.resolved!.url, dest) + } + + // TODO: return skipped/failed packages back to the caller + async function resolvePackageManifest(manifest: TerraformPackageManifest): Promise> { + const maps: Record = {} + + getLogger().emitInstallEvent({ + phase: 'download', + packages: Object.values(manifest.packages).map(v => v.resolved!.url), + } as any) + + const packages = Object.fromEntries( + await Promise.all(Object.entries(manifest.packages).map(async ([k, v]) => [k, await ensurePackage(v)] as const)) + ) + + const failed = Object.entries(packages).filter(([k, v]) => v.type === 'err').map(([k, v]) => { + const pkg = manifest.packages[k] + + return new Error(`Failed to install package "${pkg.name}-${pkg.version}"`, { + cause: (v as PackageInstallResult & { type: 'err' }).reason + }) + }) + + if (failed.length > 0) { + throw new AggregateError(failed) + } + + manifest.dependencies[-1] = manifest.roots + + for (const [k, v] of Object.entries(manifest.dependencies)) { + maps[k as any] ??= {} + for (const [k2, v2] of Object.entries(v)) { + const r = packages[v2.package] + const pkg = manifest.packages[v2.package] + + if (!r) { + if (!pkg) { + throw new Error(`Missing package with id: ${v2.package}`) + } + throw new Error(`Missing package: ${pkg.name}-${pkg.version}`) + } + + // Dead code, only done to appease CFA + if (r.type === 'err') { + throw r.reason + } + + if (r.type === 'skipped') { + getLogger().debug(`Skipped package "${pkg.name}-${pkg.version}":`, r.reason) + continue + } + + maps[k as any][k2] = { + mapping: v2.package, + location: r.destination, + versionConstraint: v2.versionConstraint, + source: { type: 'package', data: pkg }, + } + } + } + + return maps + } + + async function createImportMap(manifest: TerraformPackageManifest, maps?: Record): Promise { + const resolved = maps ?? await resolvePackageManifest(manifest) + const trees = new Map() + const maps2: Record = {} + function resolveTree(tree = manifest.roots, id = -1): ImportMap2 { + const map: ImportMap2 = maps2[id] ??= {} + const m = resolved[id] + for (const [k, v] of Object.entries(tree)) { + const o = m[k] + if (!o) { + // This can happen when skipping a package download + continue + } + + const pkgId = typeof v === 'number' ? v : v.package + const key = String(pkgId) + if (trees.has(key)) { + map[k] = trees.get(key)! + } else { + map[k] = { + location: o.location, + source: { type: 'package', data: manifest.packages[pkgId] }, + } + trees.set(key, map[k]) + + const deps = manifest.dependencies[pkgId] + if (deps) { + (map[k] as Mutable).mapping = resolveTree(deps, o.mapping) + } + } + } + return map + } + return resolveTree() + } + + const hasher = createHasher() + const importMaps = new WeakMap>() + + function getImportMap(deps: TerraformPackageManifest) { + if (importMaps.has(deps)) { + return importMaps.get(deps)! + } + + const key = hasher.hash(deps) + const dest = path.resolve(mappingCacheDir, key) + const result = worker() + importMaps.set(deps, result) + + return result + + async function worker(): Promise { + try { + const data = JSON.parse(await fs.readFile(dest, 'utf-8')) + + return createImportMap(deps, data) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + getLogger().warn('Evicting invalid import from cache due to error:', e) + + importMaps.delete(deps) + await fs.deleteFile(dest) + } + + const m = await resolvePackageManifest(deps) + const [r] = await Promise.all([ + createImportMap(deps, m), + fs.writeFile(dest, JSON.stringify(m)) + ]) + + return r + } + } + } + + async function resolveProviderConfig(provider: ProviderConfig): Promise { + const generator = await getProviderGenerator() + const resolved = await generator.resolveProvider(provider) + + return { + name: resolved.name, + type: 'synapse-provider', + version: resolved.version, + resolved: { + url: resolved.source, + } + } + } + + async function installSynapseProvider(provider: ProviderConfig) { + const info = await resolveProviderConfig(provider) + const res = await ensurePackage(info) + if (res?.type !== 'ok') { + throw res?.reason ?? new Error(`Failed to install provider "${provider.name}"`) + } + + return { + info, + destination: res.destination, + } + } + + async function installProviderTypes(dir: string, pkgs: Record) { + getLogger().log(`Installing types`) + const gen = await getProviderGenerator() + await gen.installTypes(pkgs, dir) + } + + async function downloadPackage(name: string, type: PackageInfo['type']) { + if (type === 'synapse-provider') { + await installSynapseProvider({ name }) + } else { + throw new Error(`Not implemented: ${type}`) + } + } + + + //const pkgData = pkgFiles[0].contents + //const pkg = JSON.parse(pkgData.toString('utf-8')) as PackageJson + + // const postinstall = pkg.scripts?.postinstall + // if (postinstall) { + // if (runInstallScripts) { + // getLogger().log(`Running postinstall script for package: ${info.resolved!.url}`) + // await runPostInstall(postinstall, dest, pkg) + // } else { + // getLogger().warn(`Skipping running postinstall script for package: ${info.resolved!.url}`) + // } + // } + + async function getImportMapForPackage(pkg: PackageJson) { + const deps = { ...pkg.dependencies, ...pkg.optionalDependencies } + const repo = getNpmPackageRepo() + const result = await resolveDeps(deps, repo) + const manifest = await toPackageManifest(repo, result) + + return getImportMap(manifest) + } + + async function runPostInstall(postinstall: string, pkgDir: string, pkg: PackageJson) { + const importMap = await getImportMapForPackage(pkg) + const importMapPath = path.resolve(pkgDir, 'import-map.json') + await fs.writeFile(importMapPath, JSON.stringify(importMap)) + + // XXX: not really used, need to wrap `child_process` to + // forward to a virtualized machine to make this work well + const targetEnv = { + ...process.env, + 'NODE_ARCH': nodeArch, + 'NODE_PLATFORM': nodePlatform, + 'NODE_ENDIANNESS': resolvedTarget.endianness, + } + + // const result = await execCommand(postinstall, { + // env: await resolveWrapperEnvVars(fs, importMapPath, targetEnv), + // cwd: pkgDir, + // }).finally(() => fs.deleteFile(importMapPath)) + } + + async function getPublishedMappings(name: string, pointer: string): Promise<[string, ImportMap2[string]] | undefined> { + const { hash, storeHash } = getPointerComponents(pointer) + if (!storeHash) { + //throw new Error(`Malformed pointer: ${v}`) + getLogger().warn(`Ignoring old published file: ${name}`) + return + } + + return [`${pointerPrefix}${hash}`, { + source: { type: 'artifact', data: { hash: hash, metadataHash: storeHash } }, + location: `${pointerPrefix}${storeHash}:${hash}`, + locationType: 'module', + mapping: await getPointerMappings(hash, storeHash), + }] + } + + async function getPublishedMappings2(name: string, pointer: string): Promise<[string, ImportMap2[string]] | undefined> { + const { hash, storeHash } = getPointerComponents(pointer) + if (!storeHash) { + //throw new Error(`Malformed pointer: ${v}`) + getLogger().warn(`Ignoring old published file: ${name}`) + return + } + + const m = await getPointerMappings(hash, storeHash) + if (!m) { + return + } + + return Object.entries(m)[0] + } + + const pointerMappings = new Map | undefined>() + async function getPointerMappings(hash: string, storeHash: string) { + const cacheKey = `${storeHash}:${hash}` + if (pointerMappings.has(cacheKey)) { + return pointerMappings.get(cacheKey) + } + + const m = repo.getMetadata(hash, storeHash) + if (!m.dependencies && !m.packageDependencies) { + pointerMappings.set(cacheKey, undefined) + + return + } + + const mapping: ImportMap = { + ...(m.packageDependencies ? await getImportMap(m.packageDependencies) : undefined) + } + + // TODO: we can do the same thing with `pointers` to avoid having to load anything at runtime + if (m.dependencies) { + for (const [k, v] of Object.entries(m.dependencies)) { + for (const k2 of v) { + mapping[`${pointerPrefix}${k2}`] = { + source: { type: 'artifact', data: { hash: k2, metadataHash: k }}, + location: `${pointerPrefix}${k}:${k2}`, + locationType: 'module', + mapping: await getPointerMappings(k2, k) + } + } + } + } + + pointerMappings.set(cacheKey, mapping) + + return mapping + } + + return { + getImportMap, + installProviderTypes, + downloadSynapseTool, + downloadPackage: downloadPackage, + + getPublishedMappings, + getPublishedMappings2, + getPointerMappings, + } +} + +async function loadSynapsePackage(repo: DataRepository, pkgDir: string, snapshot: Snapshot, k: string, v: NpmPackageInfo) { + // TODO: symlinked packages that are not overriden need to dump their `.d.ts` files + + const targets: TargetsFile = {} + const published: Record = {} + const runtimeModules: Snapshot['moduleManifest'] = {} + const pointers: Record> = {} + + if (snapshot.targets) { + for (const [k2, v2] of Object.entries(snapshot.targets)) { + const targetModule = k2.startsWith('./') ? path.resolve(pkgDir, k2) : k2 + targets[targetModule] = v2 + } + } + + if (snapshot.moduleManifest) { + for (const [k, v] of Object.entries(snapshot.moduleManifest)) { + runtimeModules[k] = { + types: undefined, + ...v, + path: path.resolve(pkgDir, v.path), + } + } + } + + if (snapshot.published) { + for (const [k, v] of Object.entries(snapshot.published)) { + const [storeHash, hash] = v.split(':') + if (!hash) { + throw new Error(`Malformed pointer: ${v}`) + } + published[path.resolve(pkgDir, k)] = createPointer(hash, storeHash) + } + } + + if (snapshot.pointers) { + for (const [k, v] of Object.entries(snapshot.pointers)) { + pointers[path.resolve(pkgDir, k)] = v + } + } + + // We apply the mount _relative_ to the target directory + const store = snapshot.store ?? (await repo.getBuildFs(snapshot.storeHash)) + const spec = v.specifier ?? v.name + + return { + spec, + store, + targets, + pointers, + published, + runtimeModules, + } +} + +function getPackages(installation: Pick, workingDir = getWorkingDir()) { + return Object.fromEntries(Object.entries(installation.packages).map(([k, v]) => { + return [path.resolve(workingDir, k), { + ...v, + directory: path.resolve(workingDir, v.directory), + }] as const + })) +} + +const getSnapshot = keyedMemoize(loadSnapshot) + +export async function loadTypes() { + const workingDirectory = getWorkingDirectory() + const installation = await getInstallation(getProgramFs()) + const types: Record = {} + const runtimeModules: Record = {} + const packages = installation?.packages + + if (!packages) { + getLogger().warn(`No packages installed, type generation may not be correct`) + + return { types, runtimeModules } + } + + for (const [k, v] of Object.entries(packages)) { + const pkgDir = path.resolve(workingDirectory, v.directory) + const installedDir = path.resolve(workingDirectory, k) + if (v.types) { + const runtimeManifest = v.snapshot?.moduleManifest + if (runtimeManifest) { + for (const [k, v] of Object.entries(runtimeManifest)) { + runtimeModules[k] = resolveRelative(installedDir, v.path).replace(/\.js$/, '.d.ts') + } + } + + types[installedDir] = v.types + continue + } + + if (!v.packageFile.synapse || pkgDir === workingDirectory) { + continue + } + + const snapshot = v.snapshot ?? await getSnapshot(pkgDir) + if (!snapshot) { + // throw new Error(`Missing build artifacts for package: ${k}`) + getLogger().debug(`No snapshot found for "${k}"`) + continue + } + + if (snapshot.types) { + types[installedDir] = snapshot.types + } + + if (snapshot.moduleManifest) { + for (const [k, v] of Object.entries(snapshot.moduleManifest)) { + runtimeModules[k] = resolveRelative(installedDir, v.path).replace(/\.js$/, '.d.ts') + } + } + } + + return { types, runtimeModules } +} + +interface DeferredTargets { + readonly specifier: string + readonly data: TargetsFile +} + +export function resolveDeferredTargets(moduleResolver: ModuleResolver, targets: DeferredTargets[]): TargetsFile { + const res: TargetsFile = {} + for (const t of targets) { + // TODO: implement importer + const entrypoint = moduleResolver.resolveVirtual(t.specifier) + + for (const [k, v] of Object.entries(t.data)) { + res[k] = v + + for (const symbolInfo of Object.values(v)) { + for (const [targetName, mapping] of Object.entries(symbolInfo)) { + if (isRelativeSpecifier(mapping.moduleSpecifier)) { + symbolInfo[targetName] = { + ...symbolInfo[targetName], + moduleSpecifier: moduleResolver.resolveVirtual(mapping.moduleSpecifier, entrypoint), + } + } + } + } + } + } + + // Final target resolution + for (const [k, v] of Object.entries(res)) { + res[moduleResolver.resolve(k)] = v + } + + return res +} + + +function getPointerComponents(pointer: string): { hash: string; storeHash?: string } { + if (isDataPointer(pointer)) { + return pointer.resolve() + } + + if (pointer.startsWith(pointerPrefix)) { + pointer = pointer.slice(pointerPrefix.length) + } + + const [storeHash, hash] = pointer.split(':') + if (!hash) { + return { hash: storeHash } + } + + return { hash, storeHash } +} + + +// JSR +// https://npm.jsr.io +// jsr:@luca/cases -> @jsr/luca__cases +// NOTE: All requests to the JSR registry API must be made with an Accept header that does not include text/html, and requests must not specify Sec-Fetch-Dest: document. When fetching with Accept: text/html, the registry may return an HTML page with a rendered version of the underlying data. + +const installations = new Map, ReturnType>() +function getInstallationCached(fs: Pick) { + if (installations.has(fs)) { + return installations.get(fs)! + } + + const res = getInstallation(fs) + installations.set(fs, res) + + return res +} + +export type PackageService = Awaited> +export async function createPackageService(moduleResolver: ModuleResolver, repo = getDataRepository(), programFs: Pick & Pick = getProgramFs()) { + const workingDirectory = getWorkingDirectory() + + const fs = getFs() + const installer = createPackageInstaller({ + fs, + getProviderGenerator: memoize(async () => { + const tfPath = await getTerraformPath() + + return createProviderGenerator(fs, providerRegistryHostname, tfPath) + }) + }) + + function registerRuntimeMapping(modules: Record>, dir: string) { + const resolved = Object.fromEntries(Object.entries(modules).map(([k, v]) => [k, path.resolve(dir, v.path)])) + moduleResolver.registerGlobals(resolved) + getLogger().debug(`Registered globals`, resolved) + } + + async function loadProgramState() { + const [installation, runtimeModules = {}, infraMappings, currentPointers] = await Promise.all([ + getInstallationCached(programFs), + // We used to load runtime modules contributed by the current build target + // But now we expect them to only come from external packages + // getModuleMappings(programFs), + undefined as Snapshot['moduleManifest'], + readInfraMappings(programFs), + readPointersFile(programFs), + ]) + + return { + installation, + runtimeModules, + infraMappings, + currentPointers, + } + } + + async function loadIndex() { + const { installation, runtimeModules, infraMappings, currentPointers } = await loadProgramState() + if (!installation?.packages) { + throw new Error(`No packages installed`) + } + + const packageIds = installation.importMap?.sources + ? new Map(Object.entries(installation.importMap.sources).filter(x => !!x[1]).map(x => [x[1], x[0]] as const)) + : undefined + + if (installation.importMap) { + const expanded = expandImportMap(installation.importMap) + moduleResolver.registerMapping(expanded, workingDirectory) + } + + const infraFiles = Object.fromEntries( + Object.entries(infraMappings).map(([k, v]) => [path.resolve(workingDirectory, k), path.resolve(workingDirectory, v)]) + ) + + const packages = runTask('get', 'packages (index)', () => getPackages(installation), 1) + const stores: Record = {} + const pointers: Record> = {} + + const deferredTargets: DeferredTargets[] = [] + + if (currentPointers) { + for (const [k, v] of Object.entries(currentPointers)) { + pointers[path.resolve(workingDirectory, k)] = v + } + } + + async function setupSynapsePkg(k: string, v: (typeof packages)[string]) { + const pkgDir = path.resolve(workingDirectory, v.directory) + const snapshot = v.snapshot ?? await getSnapshot(pkgDir) + if (!snapshot) { + // throw new Error(`Missing build artifacts for package: ${k}`) + getLogger().debug(`No snapshot found for "${k}"`) + return + } + + const res = await loadSynapsePackage(repo, pkgDir, snapshot, k, v) + stores[path.relative(workingDirectory, pkgDir)] = res.store + + for (const [k, v] of Object.entries(res.pointers)) { + pointers[k] = v + } + + for (const [k, v] of Object.entries(res.published)) { + await registerPointerDependencies(v, k) + } + + for (const [k, v] of Object.entries(res.runtimeModules)) { + if (runtimeModules[k] && runtimeModules[k].path !== v.path) { + // throw new Error(`Conflicting runtime module found for "${k}": ${resolved} conflicts with ${runtimeModules[k].path}`) + } else { + runtimeModules[k] = v as any + } + } + + deferredTargets.push({ + specifier: res.spec, + data: res.targets, + }) + } + + const p: Promise[] = [] + for (const [k, v] of Object.entries(packages)) { + if ((!v.isSynapsePackage && !v.packageFile.synapse) || path.resolve(workingDirectory, v.directory) === workingDirectory) { + continue + } + + p.push(runTask('synapse-pkg', k, () => setupSynapsePkg(k, v), 1).catch(e => { + throw new Error(`Failed to load package "${k}"`, { cause: e }) + })) + } + + await Promise.all(p) + + registerRuntimeMapping(runtimeModules, workingDirectory) + const pkgResolver = createPackageResolver( + fs, + workingDirectory, + moduleResolver, + packages, + packageIds as Map + ) + + const runtimeMappings = Object.fromEntries(Object.entries(runtimeModules).map(([k, v]) => [k, path.resolve(workingDirectory, v.path)])) + + return { stores, packages, infraFiles, pkgResolver, pointers, runtimeMappings, deferredTargets, importMap: installation.importMap } + } + + async function registerPointerDependencies(pointer: string, name?: string) { + if (!isDataPointer(pointer)) { + throw new Error(`Not implemented: ${pointer}`) + } + + if (name) { + const mapping = await installer.getPublishedMappings2(name, pointer) + if (mapping) { + moduleResolver.registerMapping(Object.fromEntries([mapping])) + getLogger().debug(`Registered published import map for module "${name}":`, pointer) + } + + return + } + + const { hash, storeHash } = pointer.resolve() + const mapping = await installer.getPointerMappings(hash, storeHash) + if (mapping) { + moduleResolver.registerMapping(mapping, pointer) + getLogger().debug('Registered import map for module:', pointer, Object.keys(mapping)) + } + } + + return { + getPublishedMappings: installer.getPublishedMappings, + getPublishedMappings2: installer.getPublishedMappings2, + downloadPackage: installer.downloadPackage, + getImportMap: installer.getImportMap, + loadIndex, + registerPointerDependencies, + downloadSynapseTool: installer.downloadSynapseTool, + } +} + +type PackageInstallResult = { + type: 'ok' + name: string + destination: string + cacheHit: boolean +} | { + type: 'err' + reason: Error +} | { + type: 'skipped' + reason: string +} + +// `browser`? `react-native`? +const partialPackageKeys = ['name', 'main', 'bin', 'bundledDependencies', 'optionalDependencies', 'dependencies', 'exports', 'module', 'devDependencies', 'peerDependencies', 'peerDependenciesMeta', 'synapse', 'type', 'scripts'] as const +const lifecycleScripts = ['postinstall'] + +type PartialPackageJson = Pick + +function prunePkgJsonScripts(pkgScripts: Record): Record | undefined { + let didAdd = false + const res: any = {} + for (const k of lifecycleScripts) { + res[k] = pkgScripts[k] + didAdd ||= !!pkgScripts[k] + } + return didAdd ? res : undefined +} + +function prunePkgJson(pkg: PackageJson): PartialPackageJson { + const res: any = {} + for (const k of partialPackageKeys) { + res[k] = pkg[k] + if (k === 'scripts' && res[k]) { + res[k] = prunePkgJsonScripts(res[k]) + } + } + return res +} + +export interface NpmPackageInfo { + readonly name: string + readonly version?: string + readonly directory: string + readonly packageFile: PartialPackageJson + readonly resolved?: string + readonly integrity?: string + + // Synapse only + readonly types?: TypesFileData + readonly specifier?: string + readonly snapshot?: Snapshot + + // readonly hasInstallScript?: boolean +} + +interface LookupResult { + readonly module: string + readonly packageId?: string + + // old + readonly location?: string + readonly packageInfo?: PackageInfo + readonly dependencies?: DependencyTree + readonly versionConstraint?: string +} + +interface FlatImportMap { + [specifier: string]: { + readonly location: string + readonly mapping?: number + readonly source?: SourceInfo + readonly versionConstraint: string + } +} + +// This omits any nodes that _are not_ in `keys` +export function pruneManifest(manifest: TerraformPackageManifest, keys: Iterable): TerraformPackageManifest { + const m: TerraformPackageManifest = { roots: {}, dependencies: {}, packages: {} } + + function insert(id: number, stack: number[] = []) { + if (id in m.packages) return + + const pkg = m.packages[id] = manifest.packages[id] + const pkgDeps = (pkg as any).dependencies + const pkgPeerDeps = (pkg as any).peers + const nestedDeps = manifest.dependencies[id] + + if (nestedDeps) { + m.dependencies[id] = nestedDeps + + for (const v of Object.values(nestedDeps)) { + insert(v.package, [...stack, id]) + } + } + + const deps = { ...pkgDeps, ...pkgPeerDeps } + for (const k of Object.keys(deps)) { + // for (let i = stack.length - 1; i >= 0; i--) { + // const peers = manifest.dependencies[stack[i]] + // } + if (manifest.roots[k]) { + m.roots[k] = manifest.roots[k] + insert(m.roots[k].package) + } + } + } + + for (const k of keys) { + const r = manifest.roots[k] + if (r === undefined || k in m.roots) continue + + m.roots[k] = manifest.roots[k] + insert(m.roots[k].package) + } + + return m +} + diff --git a/src/pm/publish.ts b/src/pm/publish.ts new file mode 100644 index 0000000..0ce8ce3 --- /dev/null +++ b/src/pm/publish.ts @@ -0,0 +1,868 @@ +import * as path from 'node:path' +import { StdioOptions } from 'node:child_process' +import { mergeBuilds, pruneBuild, consolidateBuild, commitPackages, getInstallation, writeSnapshotFile, getProgramFs, getDataRepository, getModuleMappings, loadSnapshot, dumpData, getProgramFsIndex, getDeploymentFsIndex, toFsFromIndex, copyFs, createSnapshot, getOverlayedFs, Snapshot, ReadonlyBuildFs } from '../artifacts' +import { NpmPackageInfo, getDefaultPackageInstaller, installFromSnapshot, testResolveDeps } from './packages' +import { getBinDirectory, getSynapseDir, getLinkedPackagesDirectory, getToolsDirectory, getUserEnvFileName, getWorkingDir, getWorkingDirectory, listPackages, resolveProgramBuildTarget, SynapseConfiguration, getUserSynapseDirectory, setPackage, BuildTarget, findDeployment } from '../workspaces' +import { gzip, isNonNullable, keyedMemoize, linkBin, makeExecutable, memoize, throwIfNotFileNotFoundError, tryReadJson } from '../utils' +import { Fs, ensureDir } from '../system' +import { glob } from '../utils/glob' +import { createTarball, extractTarball } from '../utils/tar' +import { getLogger, runTask } from '..' +import { homedir } from 'node:os' +import { getBuildTargetOrThrow, getFs, getSelfPathOrThrow, isSelfSea } from '../execution' +import { ImportMap, expandImportMap, hoistImportMap } from '../runtime/importMaps' +import { createCommandRunner, patchPath, runCommand } from '../utils/process' +import { PackageJson, ResolvedPackage, getImmediatePackageJsonOrThrow, getPackageJson } from './packageJson' +import { readKey, setKey } from '../cli/config' +import { getEntrypointsFile } from '../compiler/programBuilder' +import { createPackageForRelease } from '../cli/buildInternal' + +const getDependentsFilePath = () => path.resolve(getUserSynapseDirectory(), 'packageDependents.json') + +interface DependentsData { + [directory: string]: Record +} + +async function _getDependentsData(): Promise { + return getFs().readFile(getDependentsFilePath(), 'utf-8').then(JSON.parse).catch(e => { + throwIfNotFileNotFoundError(e) + + return {} + }) +} + +export const getDependentsData = memoize(_getDependentsData) + +export async function setDependentsData(data: DependentsData): Promise { + await getFs().writeFile(getDependentsFilePath(), JSON.stringify(data, undefined, 4)) +} + +export function updateDependent(dependents: DependentsData[string], key: string, location: string, hash: string) { + dependents[key] = { location, hash, timestamp: new Date().toISOString() } + const size = Object.values(key).length + if (size > 5) { + const entries = Object.entries(dependents).map(([k, v]) => [k, v.timestamp ? new Date(v.timestamp).getTime() : Date.now()] as const) + entries.sort((a, b) => b[1] - a[1]) + + const rem = entries.map(x => x[0]).slice(4) + for (const k of rem) { + delete dependents[k] + } + } +} + +async function getDependents(pkgLocation: string) { + const data = await getDependentsData() + + return data[pkgLocation] ?? {} +} + +async function publishPkgUpdates(pkgLocation: string, snapshot: Snapshot & { store: ReadonlyBuildFs }, fs = getFs()) { + let needsUpdate = false + const dependents = await getDependents(pkgLocation) + for (const [k, v] of Object.entries(dependents)) { + if (v.hash === snapshot.storeHash) continue + + if (!(await fs.fileExists(v.location))) { + getLogger().debug(`Removing stale dependent "${v.location}"`) + + delete dependents[k] + needsUpdate = true + continue + } + + getLogger().debug(`Updating dependent at "${v.location}"`) + await installFromSnapshot(k, v.location, pkgLocation, snapshot) + needsUpdate = true + updateDependent(dependents, k, v.location, snapshot.storeHash) + } + + if (needsUpdate) { + await setDependentsData({ + ...(await getDependentsData()), + [pkgLocation]: dependents, + }) + } +} + +function getLinkedPkgPath(name: string, deploymentId?: string) { + const packagesDir = getLinkedPackagesDirectory() + + return path.resolve(packagesDir, deploymentId ? `${name}-${deploymentId}` : name) +} + +export async function linkPackage(opt?: PublishOptions & { globalInstall?: boolean; skipInstall?: boolean; useNewFormat?: boolean }) { + const bt = getBuildTargetOrThrow() + + const packageDir = opt?.packageDir ?? getWorkingDir() + const pkg = await getPackageJson(getProgramFs(), packageDir, false) + if (!pkg) { + throw new Error(`No "package.json" found: ${packageDir}`) + } + + const fs = getFs() + const pkgName = pkg.data.name ?? path.basename(pkg.directory) + const resolvedDir = getLinkedPkgPath(pkgName, bt.deploymentId) + if (opt?.useNewFormat) { + return createPackageForRelease(packageDir, resolvedDir, undefined, true, true) + } + + const pruned = await createMergedView(bt.programId, bt.deploymentId) + await copyFs(pruned, resolvedDir) + await patchSourceRoots(bt.workingDirectory, resolvedDir) + const { snapshot, committed } = await createSnapshot(pruned, bt.programId, bt.deploymentId) + await writeSnapshotFile(fs, resolvedDir, snapshot) + await dumpData(resolvedDir, pruned, snapshot.storeHash) + await dumpData(resolvedDir, pruned, snapshot.storeHash, true) // Write a block too + + if (!pruned.files['package.json']) { + await fs.writeFile( + path.resolve(resolvedDir, 'package.json'), + await fs.readFile(path.resolve(packageDir, 'package.json')) + ) + } else { + await fs.writeFile( + path.resolve(resolvedDir, 'package.json'), + await getDataRepository().readData(pruned.files['package.json'].hash) + ) + } + + await setPackage(pkgName, bt.programId) + + if (pkgName === 'synapse') { + for (const [k, v] of Object.entries(getPkgExecutables(pkg.data) ?? {})) { + await makeExecutable(path.resolve(resolvedDir, v)) + } + + if (!opt?.skipInstall) { + await writeImportMap(resolvedDir, snapshot.published, snapshot.storeHash) + } + + // XXX: remove deps + const pkgData = JSON.parse(await fs.readFile(path.resolve(resolvedDir, 'package.json'), 'utf-8')) + delete pkgData.bin + delete pkgData.dependencies + await fs.writeFile(path.resolve(resolvedDir, 'package.json'), JSON.stringify(pkgData)) + } + + async function replaceIntegration(name: string) { + const overrides = await readKey>('projectOverrides') ?? {} + overrides[`synapse-${name}`] = resolvedDir + await setKey('projectOverrides', overrides) + } + + switch (pkgName) { + case 'synapse-local': + await replaceIntegration('local') + } + + + // Used internally for better devex + // This can make things really slow if there are many dependents + if (!bt.environmentName) { + const snapshotWithStore = Object.assign(snapshot, { store: committed }) + await publishPkgUpdates(resolvedDir, snapshotWithStore) + await publishPkgUpdates(packageDir, snapshotWithStore) + } + + return resolvedDir +} + +export async function emitPackageDist(dest: string, bt: BuildTarget, tsOutDir?: string, declaration?: boolean) { + const pruned = await createMergedView(bt.programId, bt.deploymentId) + + if (!declaration) { + for (const [k, v] of Object.entries(pruned.files)) { + if (k.endsWith('.d.ts') || k.endsWith('.d.ts.map')) { + delete pruned.files[k] + } + } + } + + // We only want files under `tsOutDir` + if (tsOutDir) { + for (const [k, v] of Object.entries(pruned.files)) { + delete pruned.files[k] + if (!k.startsWith(tsOutDir)) { + continue + } + + pruned.files[path.posix.relative(tsOutDir, k)] = v + } + } + + await copyFs(pruned, dest, undefined, false) + + return dest +} + +export async function dumpPackage(dest: string, opt?: { debug?: boolean }) { + const fs = getFs() + const pkg = await getImmediatePackageJsonOrThrow() + const bt = getBuildTargetOrThrow() + + const programId = bt.programId + const deploymentId = bt.deploymentId + + const pruned = await createMergedView(programId, deploymentId) + + await copyFs(pruned, dest) + const { snapshot } = await createSnapshot(pruned, programId, deploymentId) + await writeSnapshotFile(fs, dest, snapshot) + + if (!pruned.files['package.json']) { + await fs.writeFile( + path.resolve(dest, 'package.json'), + await fs.readFile(path.resolve(getWorkingDir(), 'package.json')) + ) + } + + for (const [k, v] of Object.entries(getPkgExecutables(pkg.data) ?? {})) { + await makeExecutable(path.resolve(dest, v)) + } + + await writeImportMap(dest, snapshot.published, snapshot.storeHash) + + return dest +} + +export function getPkgExecutables(pkgData: PackageJson) { + const bin = pkgData.bin + if (!bin) { + return + } + + if (typeof bin === 'string' && !pkgData.name) { + throw new Error('An executable name must be provided when omitting the package name') + } + + return typeof bin === 'string' ? Object.fromEntries([[pkgData.name, bin]]) : bin +} + +async function getOverridesFromProject() { + const res: Record = {} + const bt = getBuildTargetOrThrow() + const projectId = bt.projectId + const packages = await listPackages(projectId) + for (const [k, v] of Object.entries(packages)) { + const procId = await findDeployment(v, projectId, bt.environmentName) + res[k] = getLinkedPkgPath(k, procId) + } + + return res +} + +async function getMergedOverrides() { + const [fromConfig, fromProject] = await Promise.all([ + readKey>('projectOverrides'), + getOverridesFromProject() + ]) + + return { ...fromProject, ...fromConfig } +} + +async function _getLocalOverrides() { + const overrides = await getMergedOverrides() + + return Object.keys(overrides).length === 0 ? undefined : overrides +} + +const getLocalOverrides = memoize(_getLocalOverrides) + +export async function getProjectOverridesMapping(fs: Fs) { + const overrides = await getLocalOverrides() + if (!overrides) { + return + } + + const mapping: Record = {} + for (const [k, v] of Object.entries(await listPackages())) { + const dest = overrides[k] + if (!dest) { + continue + } + + mapping[dest] = getWorkingDir(v) + } + + return mapping +} + +export async function getSelfDir() { + const pkg = await getPackageJson(getFs(), path.dirname(getSelfPathOrThrow()), true) + if (!pkg || pkg.data.name !== 'synapse') { + return + } + + return pkg.directory +} + +async function findOwnSnapshot(currentDir: string) { + const fs = getFs() + const pkg = await getPackageJson(fs, currentDir, true) + if (!pkg) { + getLogger().error(`Failed to find own package starting from ${currentDir}`) + return + } + + const snapshot = await loadSnapshot(pkg.directory) + if (!snapshot) { + throw new Error(`Missing own snapshot`) + } + + return { pkg, snapshot } +} + +export async function addImplicitPackages(packages: Record, synapseConfig?: SynapseConfiguration) { + const withTargets = synapseConfig?.target && !synapseConfig?.sharedLib + ? maybeAddTargetPackage(packages, synapseConfig.target) + : packages + + // Already installed as a project package + const s = Object.values(packages).find(x => x === `spr:#synapse`) + if (s) { + return withTargets + } + + // The time is from copying files. Only for "block-less" copies. + // When using blocks this takes less than a millisecond + // + // 2024-04-05T20:23:03.150Z [PERF] packages (findOwnSnapshot) 944.399 ms + const _findOwnSnapshot = () => runTask('packages', 'findOwnSnapshot', () => findOwnSnapshot(path.dirname(getSelfPathOrThrow())), 1) + + async function getSelfDir() { + return (await getPackageOverride('synapse')) ?? (await _findOwnSnapshot())?.pkg.directory + } + + const selfDir = await getSelfDir() + if (!selfDir) { + throw new Error(`Failed to find Synapse runtime package`) + } + + getLogger().debug('Adding self to package dependencies') + + return { + ...withTargets, + '@cohesible/synapse': `file:${selfDir}`, + } +} + +function maybeAddTargetPackage(packages: Record, target: string) { + const pkgName = `synapse-${target}` + const pkgUrl = `spr:#${pkgName}` + + // Already installed as a project package + const s = Object.values(packages).find(x => x === pkgUrl) + if (s) { + return packages + } + + getLogger().debug('Adding target to package dependencies') + + return { + ...packages, + [`@cohesible/${pkgName}`]: pkgUrl, + } +} + +export async function getPackageOverride(spec: string) { + const overrides = await getLocalOverrides() + + return overrides?.[spec] +} + +interface CachedImportMap { + mappings: ImportMap + fsHash: string +} + +const importMapFileName = '[#install]__import-map__.json' +async function readCachedImportMap() { + const programFs = getProgramFs() + const d = await tryReadJson(programFs, importMapFileName) + if (d && !('fsHash' in d)) { + return + } + + return d +} + +async function writeCachedImportMap(mapping: CachedImportMap) { + const programFs = getProgramFs() + + await programFs.writeFile(importMapFileName, JSON.stringify(mapping)) +} + +async function writeImportMap(packageDir: string, published?: Record, fsHash?: string) { + const fs = getFs() + const cached = fsHash ? await readCachedImportMap() : undefined + if (cached && cached.fsHash === fsHash) { + await fs.writeFile(path.resolve(packageDir, 'import-map.json'), JSON.stringify(cached.mappings)) + + return + } + + const installation = await getInstallation(getProgramFs()) + + async function getMapping() { + if (installation?.importMap) { + return expandImportMap(installation.importMap) + } + + const m = await testResolveDeps(path.resolve(packageDir, 'package.json')) + const pkgInstaller = getDefaultPackageInstaller() + return pkgInstaller.getImportMap(m) + } + + const pkgInstaller = getDefaultPackageInstaller() + const importMap = await getMapping() + const runtimeMappings = (await getModuleMappings(getProgramFs())) ?? {} + for (const [k, v] of Object.entries(runtimeMappings)) { + importMap[k] = { location: path.resolve(packageDir, v.path), locationType: 'module' } + } + + if (published) { + for (const [k, v] of Object.entries(published)) { + const m = await pkgInstaller.getPublishedMappings(k, v) + if (m) { + importMap[m[0]] = m[1] + } + } + } + + if (fsHash) { + await writeCachedImportMap({ mappings: importMap, fsHash }) + } + + await fs.writeFile(path.resolve(packageDir, 'import-map.json'), JSON.stringify(importMap)) +} + +export async function createMergedView(programId: string, deploymentId?: string, pruneInfra = true, preferProgram = false) { + const builds = await Promise.all([ + getProgramFsIndex(programId), + deploymentId ? getDeploymentFsIndex(deploymentId) : undefined + ]) + + if (builds[1] && preferProgram) { + const workingDirectory = getWorkingDir() + const programFs = toFsFromIndex(builds[0]) + const deployables = (await getEntrypointsFile(programFs))?.deployables + const programDeployables = new Set(Object.values(deployables ?? {}).map(f => path.relative(workingDirectory, f))) + for (const [k, v] of Object.entries(builds[1].files)) { + if (builds[0].files[k] && !programDeployables.has(k)) { + delete builds[1].files[k] + } + } + } + + const merged = mergeBuilds(builds.filter(isNonNullable)) + const infraFiles = pruneInfra ? Object.keys(merged.files).filter(x => x.endsWith('.infra.js') || x.endsWith('.infra.js.map')) : [] + const privateFiles = Object.keys(merged.files).filter(x => !!x.match(/^__([a-zA-Z_-]+)__\.json$/)) // XXX + const pruned = pruneBuild(merged, ['template.json', 'published.json', 'state.json', 'packages.json', ...infraFiles, ...privateFiles]) + + return pruned +} + +async function patchSourceRoots(rootDir: string, targetDir: string) { + const sourcemaps = await glob(getFs(), targetDir, ['**/*.map'], ['node_modules']) + for (const f of sourcemaps) { + const sm = JSON.parse(await getFs().readFile(f, 'utf-8')) + const source = sm.sources[0] + if (!source || source.startsWith(rootDir)) { + continue + } + + const s = path.resolve(f, '..', source) + sm.sources[0] = path.resolve(rootDir, path.relative(targetDir, s)) + await getFs().writeFile(f, JSON.stringify(sm)) + } +} + +async function getIncludedFiles(fs: Fs, pkg: ResolvedPackage) { + const included = [...(pkg.data.files ?? ['*'])] + included.push( + 'README', + 'CHANGES', + 'CHANGELOG', + 'HISTORY', + 'LICENSE', + 'LICENCE', + 'NOTICE' + ) + + const excluded: string[] = [] + excluded.push('node_modules', '.git', '.synapse') + excluded.push('**/*.map') + + return glob(fs, pkg.directory, included, excluded) +} + + +interface PublishOptions { + readonly dryRun?: boolean + readonly packageDir?: string + readonly includeSourceMaps?: boolean +} + +interface ToolsManifest { + [directory: string]: string +} + +async function readToolsManifest(fs: Fs, name: string): Promise { + const p = path.resolve(getToolsDirectory(), 'index', `${name}.json`) + + return (await tryReadJson(fs, p)) ?? {} +} + +async function writeToolsManifest(fs: Fs, name: string, data: ToolsManifest): Promise { + const p = path.resolve(getToolsDirectory(), 'index', `${name}.json`) + + return await fs.writeFile(p, JSON.stringify(data, undefined, 4)) +} + +async function createToolWrapper(name: string, manifest: ToolsManifest, fs = getFs()) { + const cases: string[] = [] + for (const [k, v] of Object.entries(manifest).sort((a, b) => b[0].localeCompare(a[0]))) { + cases.push( +` + "${k}"*) + exec "${v}" "$@" + ;; +` + ) + } + + cases.push( +` + *) + echo "No tool installed in current directory" + exit 1 + ;; +` + ) + + const text = ` +#!/usr/bin/env bash + +case $PWD in +${cases.join('')} +esac +` + + const dest = path.resolve(getBinDirectory(), name) + await fs.writeFile(dest, text, { mode: 0o755 }) +} + +const useToolManifest = false + +async function createBinShim(fs: Fs, executablePath: string) { + const shimPath = path.resolve(getSelfPathOrThrow(), '..', 'shim.exe') + const data = await fs.readFile(shimPath) + const magic = 0x74617261 + + enum ShimType { + Native, + Interpretted, + Unknown, + } + + interface ResolvedParams { + shimType: ShimType + runtime?: string + } + + function getShimType() { + switch (path.extname(executablePath)) { + case '.bat': + case '.cmd': + case '.exe': + return ShimType.Native + + case '.js': + case '.mjs': + case '.cjs': + return ShimType.Interpretted + + default: + return ShimType.Unknown + } + } + + async function parseShebangLine() { + const text = await fs.readFile(executablePath, 'utf-8') + if (!text.startsWith('#!')) { + return + } + + const nl = text.indexOf('\n') + if (nl === -1) { + return + } + + return text.slice(0, nl) + } + + async function getResolvedSelf() { + const resolved = path.resolve(getSelfPathOrThrow(), '..', '..', 'bin', 'synapse.exe') + if (!(await fs.fileExists(resolved))) { + throw new Error(`Missing runtime executable: ${resolved}`) + } + + return resolved + } + + async function resolveParams(): Promise { + let shimType = getShimType() + if (shimType === ShimType.Native) { + return { shimType } + } + + const shebang = await parseShebangLine() + if (!shebang) { + if (shimType === ShimType.Unknown) { + return { shimType } + } + + return { + shimType, + runtime: await getResolvedSelf(), + } + } + + shimType = ShimType.Interpretted + + const parts = shebang.split(' ') + if (parts[1] === 'node') { + return { + shimType, + runtime: await getResolvedSelf(), + } + } + + if (parts.length === 1) { + // TODO: convert common paths to windows equivalents e.g. /bin/sh to bash.exe + return { + shimType, + runtime: path.resolve(parts[0]), + } + } + + const runtime = (await runCommand('where', [parts[1]])).trim() + + return { shimType, runtime } + } + + const resolved = await resolveParams() + if (resolved.shimType === ShimType.Unknown) { + throw new Error(`Failed to detect executable type`) + } + + const target = Buffer.from(executablePath, 'utf-16le') + const runtime = resolved.runtime ? Buffer.from(resolved.runtime, 'utf-16le') : undefined + + const footerSize = 8 + const payloadSize = target.byteLength + 2 + (runtime ? (runtime.byteLength + 2) : 0) + const totalSize = 1 + payloadSize + footerSize + + const buf = Buffer.allocUnsafe(data.byteLength + totalSize) + buf.set(data) + + let pos = data.byteLength + pos = buf.writeUInt8(resolved.shimType, pos) + + buf.set(target, pos) + pos += target.byteLength + pos = buf.writeUint16LE(0, pos) + + if (runtime) { + buf.set(runtime, pos) + pos += runtime.byteLength + pos = buf.writeUint16LE(0, pos) + } + + pos = buf.writeUint32LE(1 + payloadSize, pos) + pos = buf.writeUint32LE(magic, pos) + + return buf +} + +export async function installBin(fs: Fs, executablePath: string, toolName: string, dir: string, removeNested = false) { + if (useToolManifest) { + await makeExecutable(executablePath) + // FIXME: PERF: this is _very_ slow atm + await installWithToolManifest(fs, executablePath, toolName, dir, removeNested) + + return + } + + if (process.platform === 'win32') { + const dest = path.resolve(dir, 'node_modules', '.bin', toolName.replace(/\..*$/, '') + '.exe') + const shim = await createBinShim(fs, executablePath) + await fs.writeFile(dest, shim) + + return + } + + const dest = path.resolve(dir, 'node_modules', '.bin', toolName) + const link = async () => fs.link(executablePath, dest, { symbolic: true, mode: 0o755, typeHint: 'file' }) + await link().catch(async e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + + await ensureDir(dest) + await link() + }) +} + +async function installWithToolManifest(fs: Fs, executablePath: string, toolName: string, dir: string, removeNested = false) { + const m = await readToolsManifest(fs, toolName) + if (removeNested) { + for (const key of Object.keys(m)) { + if (key.startsWith(dir)) { + delete m[key] + } + } + } + + if (m[dir] !== executablePath) { + m[dir] = executablePath + await writeToolsManifest(fs, toolName, m) + await createToolWrapper(toolName, m) + } +} + +function isBinInstalled() { + const paths = process.env['PATH']?.split(':') + + return paths?.includes(getBinDirectory()) +} + +// `bash` config file predcedence: +// `.bash_profile` > `.bash_login` > `.profile` + +export function createInstallCommands(synapseDir: string, includeComments = false, completionsPath?: string) { + const maybeRelPath = synapseDir.replace(homedir(), '$HOME') + const resolvedCompletions = completionsPath ?? path.resolve(synapseDir, 'completions', 'synapse.sh') + const completions = resolvedCompletions.replace(synapseDir, '$SYNAPSE_INSTALL') + + return [ + includeComments ? `# Synapse install` : '', + `export SYNAPSE_INSTALL="${maybeRelPath}"`, + `export PATH="$SYNAPSE_INSTALL/bin:$PATH"`, + includeComments ? '# enables Synapse command line completions' : '', + `[ -f "${completions}" ] && source "${completions}"` + ].filter(x => !!x) +} + +function uninstallFromProfile(lines: string[]) { + function shouldRemove(line: string) { + if (line.startsWith('# Synapse install') || line.startsWith('# enables Synapse command line')) { + return true + } + if (line.startsWith('#')) { + return false + } + return line.includes('$SYNAPSE_INSTALL') || line.includes('SYNAPSE_INSTALL=') + } + + return lines.filter(l => !shouldRemove(l)) +} + +async function installToProfile(synapseDir: string, profileFile: string, fs = getFs(), shouldOverride = true) { + const text = await fs.readFile(profileFile, 'utf-8').catch(throwIfNotFileNotFoundError) + + const getInstallationLines = () => createInstallCommands(synapseDir, true) + + if (!text) { + await fs.writeFile(profileFile, getInstallationLines().join('\n')) + + return true + } + + async function appendInstall(lines: string[]) { + const isLastLineEmpty = !lines.at(-1)?.trim() + const isSecondLastLineEmpty = !lines.at(-2)?.trim() + const installLines = getInstallationLines() + // if (exportedPathLocation === -1) { + // lines.push(...installLines) + // } else { + // lines.splice(exportedPathLocation, 1, '', ...installLines) + // } + + if (isLastLineEmpty && isSecondLastLineEmpty) { + lines.pop() + } else if (!isLastLineEmpty) { + lines.push('') + } + + lines.push(...installLines, '') + + await fs.writeFile(profileFile, lines.join('\n')) + } + + const lines = text.split('\n') + const oldInstall = lines.findIndex(x => x.startsWith('# Synapse install') || x.includes('SYNAPSE_INSTALL=')) + if (oldInstall !== -1) { + const desiredInstallLocation = `"${synapseDir.replace(homedir(), '$HOME')}"` + if (lines.findIndex(x => !x.startsWith('#') && x.includes(`SYNAPSE_INSTALL=${desiredInstallLocation}`)) !== -1) { + return true + } + // const lastLine = lines.findIndex((x, i) => i > oldInstall && x.includes('$SYNAPSE_INSTALL')) + // if (lastLine === -1) { + // // Corrupted install? + // } else if (shouldOverride) { + // lines.splice(oldInstall, (lastLine - oldInstall) + 1, ...getInstallationLines()) + // await fs.writeFile(profileFile, lines.join('\n')) + // } + await appendInstall(uninstallFromProfile(lines)) + return true + } + + await appendInstall(lines) + return true +} + +const supportedShells = ['sh', 'bash', 'zsh'] as const + +export function isSupportedShell(shell: string): shell is (typeof supportedShells)[number] { + return supportedShells.includes(shell as any) +} + +export async function installToUserPath(target: 'sh' | 'bash' | 'zsh' = 'zsh', synapseDir = getUserSynapseDirectory()) { + switch (target) { + case 'sh': + case 'bash': + return installToProfile(synapseDir, path.resolve(homedir(), '.profile')) + case 'zsh': + return installToProfile(synapseDir, path.resolve(homedir(), '.zprofile')) + } +} + +function findBinPathsSync(pkgDir: string, fs = getFs()) { + const paths: string[] = [] + + let dir = pkgDir + while (true) { + const p = path.resolve(dir, 'node_modules', '.bin') + if (fs.fileExistsSync(p)) { + paths.push(p) + } + + const next = path.dirname(dir) + if (next === dir || next === '.') { + break + } + dir = next + } + + return paths +} + +export function createNpmLikeCommandRunner(pkgDir: string, env?: Record, stdio?: StdioOptions) { + const paths = findBinPathsSync(pkgDir) + env = patchPath(paths.join(':'), env) + + return createCommandRunner({ cwd: pkgDir, env, stdio }) +} + diff --git a/src/pm/repos/github.ts b/src/pm/repos/github.ts new file mode 100644 index 0000000..693d65a --- /dev/null +++ b/src/pm/repos/github.ts @@ -0,0 +1,107 @@ +import * as path from 'node:path' +import * as github from '../../utils/github' +import { ensureDir, keyedMemoize } from '../../utils' +import { PackageJson, getRequired } from '../packageJson' +import { PackageRepository, ResolvePatternResult } from '../packages' +import { parseVersionConstraint } from '../versions' +import { PackageInfo } from '../../runtime/modules/serdes' +import { runCommand } from '../../utils/process' +import { getFs } from '../../execution' +import { randomUUID } from 'node:crypto' +import { extractToDir, hasBsdTar } from '../../utils/tar' + +async function getPackageJsonFromRepo(owner: string, repo: string) { + const data = await github.downloadRepoFile(owner, repo, 'package.json') + + return JSON.parse(data.toString('utf-8')) as PackageJson +} + +export const githubPrefix = 'github:' + +export function createGitHubPackageRepo(): PackageRepository { + const _getPackageJson = keyedMemoize(getPackageJsonFromRepo) + + async function listVersions(name: string) { + const parsed = github.parseDependencyRef(name) + const pkg = await _getPackageJson(parsed.owner, parsed.repository) // TODO: commitish + + return [pkg.version ?? '0.0.1'] + // const releases = await github.listReleases(parsed.owner, parsed.repository) + + // return releases.map(r => r.tag_name.replace(/^v/, '')) // TODO: check semver compliance + } + + async function resolvePattern(spec: string, pattern: string): Promise { + const parsed = github.parseDependencyRef(pattern) + const pkg = await _getPackageJson(parsed.owner, parsed.repository) // TODO: commitish + + return { + name: pkg.name ?? spec, + version: parseVersionConstraint(pkg.version ?? '*'), + } + } + + async function getDependencies(name: string, version: string) { + const parsed = github.parseDependencyRef(name) + const pkg = await _getPackageJson(parsed.owner, parsed.repository) // TODO: commitish + + return getRequired(pkg) + } + + return { + listVersions, + resolvePattern, + getDependencies, + getPackageJson: async (name, version) => { + const parsed = github.parseDependencyRef(name) + const pkg = await _getPackageJson(parsed.owner, parsed.repository) // TODO: commitish + const release = await github.getRelease(parsed.owner, parsed.repository) + const asset = release.assets.find(a => a.name === `${pkg.name ?? parsed.repository}.zip`) + if (!asset) { + throw new Error(`Package not found for ${name}@${version}`) + } + + return { + ...pkg, + dist: { + integrity: '', + tarball: asset.browser_download_url, + }, + } + } + } +} + +// This does not refer to the "packages" API +export async function downloadGitHubPackage(info: PackageInfo, dest: string) { + const url = info.resolved?.url + if (!url) { + throw new Error(`Missing download URL for package: ${info.name} [destination: ${dest}]`) + } + + const data = await github.fetchData(url) + if (url.includes('.zip')) { + if (await hasBsdTar()) { + return extractToDir(data, dest, '.zip', 0) + } + + const tmp = path.resolve(path.dirname(dest), `tmp-${randomUUID()}.zip`) + await Promise.all([ + getFs().writeFile(tmp, data), + ensureDir(dest) + ]) + + try { + await runCommand('unzip', [tmp], { cwd: dest }) + } catch(e) { + await getFs().deleteFile(dest).catch() // Swallow the error, this is clean-up + throw e + } finally { + await getFs().deleteFile(tmp) + } + + return + } + + throw new Error(`Not implemented: ${url}`) +} \ No newline at end of file diff --git a/src/pm/tools.ts b/src/pm/tools.ts new file mode 100644 index 0000000..7ddabc2 --- /dev/null +++ b/src/pm/tools.ts @@ -0,0 +1,125 @@ +import { Arch, Os, toNodeArch, toNodePlatform } from '../build/builder' +import { keyedMemoize } from '../utils' +import { PackageRepository } from './packages' +import { parseVersionConstraint } from './versions' + +interface ToolVersion { + readonly os: Os + readonly arch: Arch + readonly version: string +} + +interface ToolDownloadInfo { + readonly url: string + readonly integrity?: string + readonly bin?: Record // Maps tool names to relative paths +} + +export interface ToolProvider { + readonly name: string + listVersions(): Promise + getDownloadInfo(release: ToolVersion): Promise +} + +export const toolPrefix = `synapse-tool:` + +const providers = new Map() + +export function registerToolProvider(provider: ToolProvider) { + if (provider.name.includes('/')) { + throw new Error(`Invalid character '/' in tool name`) + } + + providers.set(provider.name, provider) +} + +function getProviderOrThrow(name: string) { + const p = providers.get(name) + if (!p) { + throw new Error(`No tool provider found: ${name}`) + } + return p +} + +export function createToolRepo(): PackageRepository { + const getToolVersions = keyedMemoize((name: string) => getProviderOrThrow(name).listVersions()) + + async function listVersions(name: string) { + const m = name.match(/^(.+)\/(.+)-(.+)$/) + const versions = await getToolVersions(m?.[1] ?? name) + + if (!m) { + return Array.from(new Set(versions.map(r => r.version))) + } + + const os = m[2] as Os + const arch = m[3] as Arch + + return versions.filter(r => r.os === os && r.arch === arch).map(r => r.version) + } + + function resolvePattern(spec: string, pattern: string) { + return { name: spec, version: parseVersionConstraint(pattern) } + } + + function getName(toolName: string, os: Os, arch: Arch) { + return `${toolName}/${os}-${arch}` + } + + async function generateDeps(toolName: string, version: string) { + const versions = await getToolVersions(toolName) + const filtered = versions.filter(r => r.version === version) + + return Object.fromEntries(filtered.map(r => [`${toolPrefix}${getName(toolName, r.os, r.arch)}`, r.version])) + } + + async function getDependencies(name: string, version: string) { + const m = name.match(/^(.+)\/(.+)-(.+)$/) // this is ok because we enforce that '/' isn't allowed in tool names + if (m) { + return + } + + return generateDeps(name, version) + } + + async function getPackageJson(name: string, version: string) { + const m = name.match(/^(.+)\/(.+)-(.+)$/) + if (!m) { + return { + name, + version, + dist: { + tarball: '', + integrity: '', + isStubPackage: true, + }, + } + } + + const realName = m[1] + const os = m[2] as Os + const arch = m[3] as Arch + + const info = await getProviderOrThrow(m[1]).getDownloadInfo({ os, arch, version }) + const bin = info.bin ?? ({ [m[1]]: os !== 'windows' ? `./${realName}` : `./${realName}.exe` }) + + return { + name: realName, + version, + bin, + os: [toNodePlatform(os)], + cpu: [toNodeArch(arch)], + dist: { + tarball: info.url, + integrity: info.integrity ?? '', + } + } + } + + return { + listVersions, + resolvePattern, + getDependencies, + getPackageJson, + } +} \ No newline at end of file diff --git a/src/pm/versions.ts b/src/pm/versions.ts new file mode 100644 index 0000000..a6fded5 --- /dev/null +++ b/src/pm/versions.ts @@ -0,0 +1,251 @@ + +// Version qualifiers examples: +// 1.0, 1.0.x, ~1.0.4 - patch releases only +// 1, 1.x, ^1.0.4 - minor + patch releases +// *, x - all releases + +export type Qualifier = 'pinned' | 'patch' | 'minor' | 'all' + +export interface VersionConstraint { + pattern: string + qualifier: Qualifier + minVersion?: string + maxVersion?: string + label?: string + alt?: VersionConstraint + source?: string +} + +// TODO: `synapse add expo` +// Failed to resolve dependency flow-parser@0.* + +// TODO: handle `npm:`, `git:`, `http://` version specifiers +// https://docs.npmjs.com/cli/v10/configuring-npm/package-json#dependencies + +// https://github.com/npm/node-semver +export function parseVersionConstraint(version: string): VersionConstraint { + const source = version + + // if (version.startsWith('v')) { + // version = version.slice(1) + // } + + + if (version === '*' || version === 'x' || version === '') { + return { pattern: 'x.x.x', qualifier: 'all', minVersion: '0.0.0', source } + } + + if (version.includes('||')) { + const [left, ...rest] = version.split('||').map(x => x.trim()) + const constraint = parseVersionConstraint(left) + constraint.alt = parseVersionConstraint(rest.join(' || ')) + constraint.source = version + + return constraint + } + + if (version.startsWith('<')) { + const segments = version.slice(1).split('.') + while (segments.length < 3) { + segments.push('0') + } + return { pattern: `x.x.x`, qualifier: 'all', maxVersion: segments.join('.'), source } + } + + if (version.startsWith('>=')) { + const match = version.match(/^>=(?:[\s]*)([^\s]+) <(?:[\s]*)([^\s]+)$/) + if (match) { + // FIXME: this is not implemented entirely correct + const parts = match[1].split('.') + return { pattern: `${parts[0]}.x.x`, qualifier: 'minor', minVersion: `${parts[0]}.${parts[1] ?? 0}.${parts[2] ?? 0}`, source } + } + + const segments = version.slice(2).split('.') + while (segments.length < 3) { + segments.push('0') + } + return { pattern: `x.x.x`, qualifier: 'all', minVersion: segments.join('.'), source } + } + + if (version.includes(' - ')) { + const [left, right] = version.split(' - ').map(x => x.trim()) + const leftSegments = left.split('.') + const rightSegments = right.split('.') + + return { pattern: `x.x.x`, qualifier: 'all', minVersion: `${leftSegments[0]}.${leftSegments[1] ?? 0}.${leftSegments[2] ?? 0}`, maxVersion: `${rightSegments[0]}.${rightSegments[1] ?? 0}.${rightSegments[2] ?? 0}`} + } + + const label = version.match(/-(.*)$/)?.[1] + if (label) { + version = version.replace(`-${label}`, '') + } + + if (version.startsWith('=')) { + return { pattern: version.slice(1), qualifier: 'pinned', label, source } + } + + const segments = version.replace(/^[~^]/, '').split('.') + + if (version.startsWith('^') || segments.length === 1 || (segments.length === 2 && segments[1] === 'x')) { + if (segments.length === 1) { + return { pattern: `${segments[0]}.x.x`, qualifier: 'minor', minVersion: `${segments[0]}.0.0`, label, source } + } else if (segments.length === 2) { + return { pattern: `${segments[0]}.x.x`, qualifier: 'minor', minVersion: `${segments[0]}.${segments[1]}.0`, label, source } + } + + if (segments[0] === '0') { + if (segments[1] === '0') { + return { pattern: `${segments[0]}.${segments[1]}.${segments[2]}`, qualifier: 'pinned', label, source } + } + + return { pattern: `${segments[0]}.${segments[1]}.x`, qualifier: 'patch', minVersion: `${segments[0]}.${segments[1]}.${segments[2]}`, label, source } + } + + return { pattern: `${segments[0]}.x.x`, qualifier: 'minor', minVersion: segments.join('.'), label, source } + } + + if (version.startsWith('~') || segments.length === 2 || (segments.length === 3 && segments[2] === 'x')) { + if (segments.length === 2) { + return { pattern: `${segments[0]}.${segments[1]}.x`, qualifier: 'patch', minVersion: `${segments[0]}.${segments[1]}.0`, label, source } + } else if (segments.length === 3 && !version.startsWith('~')) { + return { pattern: `${segments[0]}.${segments[1]}.x`, qualifier: 'patch', minVersion: `${segments[0]}.${segments[1]}.0`, label, source } + } else if (segments.length === 1) { + return { pattern: `${segments[0]}.x.x`, qualifier: 'minor', minVersion: `${segments[0]}.0.0`, label, source } + } + + return { pattern: `${segments[0]}.${segments[1]}.x`, qualifier: 'patch', minVersion: segments.join('.'), label, source } + } + + return { pattern: version, qualifier: 'pinned', label, source } +} + +function getQualifierSortValue(qualifier: Qualifier): number { + switch (qualifier) { + case 'pinned': + return 0 + case 'patch': + return 1 + case 'minor': + return 2 + case 'all': + return 3 + + default: + throw new Error(`Invalid qualifier: ${qualifier}`) + } +} + +// Must be the normalized version +export function compareVersions(a: string, b: string): number { + const s1 = a.split('.') + const s2 = b.split('.') + + function cmp(a: string, b: string) { + if (a === b || a === 'x' || b === 'x') { + return 0 + } + + return Number(a) - Number(b) + } + + for (let i = 0; i < s1.length; i++) { + const v = cmp(s1[i], s2[i]) + if (v !== 0) { + return v + } + } + + return 0 +} + +function createVersionPattern(version: string) { + return new RegExp(`^${version.replace(/\./g, '\\.').replace(/x/g, '[0-9]+')}$`) +} + +function validateVersionSet(sorted: { pattern: string }[]) { + for (let i = sorted.length - 1; i > 0; i--) { + const { pattern: version } = sorted[i] + if (!createVersionPattern(version).test(sorted[i - 1].pattern)) { + throw new Error(`Incompatible versions: ${version} ∉ ${sorted[i - 1].pattern}`) + } + } +} + +export function isExact(constraint: VersionConstraint) { + return constraint.qualifier === 'pinned' && !constraint.alt +} + +export function isCompatible(a: VersionConstraint, b: VersionConstraint, ignorePrerelease = false) { + if (a.alt && isCompatible(a.alt, b)) { + return true + } + if (b.alt && isCompatible(a, b.alt)) { + return true + } + + const x = compareVersions(a.pattern, b.pattern) + if (x !== 0) { + return false + } + + // XXX: ignore pre-release tags that are equal to `0` + if (!ignorePrerelease) { + if ((a.label && a.label !== '0' && !b.label && b.source !== '*') || (!a.label && a.source !== '*' && b.label && b.label !== '0')) { + return false + } + } + + if (a.qualifier === b.qualifier) { + if (!ignorePrerelease && a.label && b.label) { + // TODO: check comparators e.g. `1.2.3-alpha.7` should satisfy `>1.2.3-alpha.3` + return a.label === b.label + } + + return true + } + + const q = getQualifierSortValue(a.qualifier) - getQualifierSortValue(b.qualifier) + if (q < 0) { + // `a` is more restrictive + // So check if the min version of b is less than or equal to a + + return compareVersions(b.minVersion ?? b.pattern, a.maxVersion ?? a.pattern) <= 0 + } + + return compareVersions(b.maxVersion ?? b.pattern, a.minVersion ?? a.pattern) >= 0 +} + +// This comparison function forms contiguous segments of incompatible constraint groups +// Each constraint in a group is sorted from least restrictive to most restrictive +export function compareConstraints(a: VersionConstraint, b: VersionConstraint, useAlt = true): number { + if (useAlt && a.alt && compareConstraints(a.alt, a, false) < 0) { + return -compareConstraints(b, a.alt) + } + + if (useAlt && b.alt && compareConstraints(b.alt, b, false) < 0) { + return compareConstraints(b.alt, a) + } + + const x = compareVersions(a.pattern, b.pattern) + if (x !== 0) { + return x + } + + if (a.qualifier === b.qualifier) { + return compareVersions(a.minVersion ?? a.pattern, b.minVersion ?? b.pattern) + } + + const q = getQualifierSortValue(a.qualifier) - getQualifierSortValue(b.qualifier) + if (q < 0) { + // `a` is more restrictive + // So check if the min version of b is less than or equal to a + + return compareVersions(a.pattern, b.minVersion ?? b.pattern) + } + + return compareVersions(a.minVersion ?? a.pattern, b.pattern) +} + +function sortConstraints(constraints: VersionConstraint[]) { + return constraints.sort(compareConstraints) +} diff --git a/src/refactoring.ts b/src/refactoring.ts new file mode 100644 index 0000000..ed9f441 --- /dev/null +++ b/src/refactoring.ts @@ -0,0 +1,1800 @@ +import type { TerraformSourceMap, Symbol, TfJson } from './runtime/modules/terraform' +import { createMinHeap, createTrie, isNonNullable, keyedMemoize, levenshteinDistance } from './utils' +import { getLogger } from '.' +import { parseModuleName } from './templates' + +interface Node { + readonly value: T + readonly children: Node[] +} + +interface TypedNode extends Node { + readonly type: U + readonly children: TypedNode[] + readonly parents?: TypedNode[] +} + +function createNode(value: T, children: Node[] = []): Node { + return { value, children } +} + +function createTypedNode(type: U, value: T): TypedNode { + return { type, value, children: [], parents: [] } +} + +// Zhang-Shasha algorithm for computing the edit distance between trees +// Reference implementation: https://github.com/timtadh/zhang-shasha/blob/master/zss/compare.py + +interface RemoveOperation { + readonly type: 'remove' + readonly node: T +} + +interface InsertOperation { + readonly type: 'insert' + readonly node: T +} + +interface UpdateOperation { + readonly type: 'update' + readonly left: T + readonly right: T +} + +interface MatchOperation { + readonly type: 'match' + readonly left: T + readonly right: T +} + +type Operation = + | RemoveOperation + | InsertOperation + | UpdateOperation + | MatchOperation + +function createAnnotatedTree(root: T) { + const lmds: number[] = [] // left-most descendents + const nodes: T[] = [] // post-order enumeration + + const ids: number[] = [] + const keyroots: Record = {} + const stack: [T, number[]][] = [[root, []]] + const pstack: [[T, number], number[]][] = [] + + //We need this when re-using nodes + //This isn't technically correct because it allows for DAGs instead of trees + // const visited = new Set() + // visited.add(root) + + let j = 0 + while (stack.length > 0) { + const [n, anc] = stack.pop()! + const nid = j + for (const c of n.children) { + // if (!visited.has(c as T)) { + // const a = [nid, ...anc] + // stack.push([c as T, a]) + // visited.add(c as T) + // } + const a = [nid, ...anc] + stack.push([c as T, a]) + } + pstack.push([[n, nid], anc]) + j += 1 + } + + let i = 0 + const tempLmds: Record = {} + while (pstack.length > 0) { + const [[n, nid], anc] = pstack.pop()! + nodes.push(n) + ids.push(nid) + + let lmd: number + if (n.children.length === 0) { + lmd = i + for (const a of anc) { + if (!(a in tempLmds)) { + tempLmds[a] = i + } else { + break + } + } + } else { + lmd = tempLmds[nid] + } + lmds.push(lmd) + keyroots[lmd] = i + i += 1 + } + + return { + root, + ids, + lmds, + nodes, + keyroots: Array.from(new Set(Object.values(keyroots))).sort((a, b) => a - b), + } +} + +// LD bounds: +// * The Levenshtein distance between two strings is no greater than the sum of their Levenshtein distances from a third string + +interface CostFunctions { + insert(value: T): number + update(oldValue: T, newValue: T): number + remove(value: T): number +} + +function editDistance(oldTree: T, newTree: T, costFn: CostFunctions) { + const a = createAnnotatedTree(oldTree) + const b = createAnnotatedTree(newTree) + + const sizeA = a.nodes.length + const sizeB = b.nodes.length + + const treeDists: number[][] = [] + const operations: Operation[][][] = [] + + for (let i = 0; i < sizeA; i++) { + treeDists[i] = [] + operations[i] = [] + + for (let j = 0; j < sizeB; j++) { + treeDists[i][j] = 0 + operations[i][j] = [] + } + } + + for (const i of a.keyroots) { + for (const j of b.keyroots) { + treedist(i, j) + } + } + + return { + cost: treeDists.at(-1)!.at(-1)!, + operations: operations.at(-1)!.at(-1)!, + } + + function treedist(i: number, j: number) { + const al = a.lmds + const bl = b.lmds + const an = a.nodes + const bn = b.nodes + + const m = i - al[i] + 2 + const n = j - bl[j] + 2 + + const fd: number[][] = [] + const partialOps: Operation[][][] = [] + for (let i2 = 0; i2 < m; i2++) { + fd[i2] = [] + partialOps[i2] = [] + for (let j2 = 0; j2 < n; j2++) { + fd[i2][j2] = 0 + partialOps[i2][j2] = [] + } + } + + const ioff = al[i] - 1 + const joff = bl[j] - 1 + + for (let x = 1; x < m; x++) { + const node = an[x + ioff] + fd[x][0] = fd[x - 1][0] + costFn.remove(node) + partialOps[x][0] = [ + ...partialOps[x - 1][0], + { type: 'remove', node } + ] + } + + for (let y = 1; y < n; y++) { + const node = bn[y + joff] + fd[0][y] = fd[0][y - 1] + costFn.insert(node) + partialOps[0][y] = [ + ...partialOps[0][y - 1], + { type: 'insert', node } + ] + } + + for (let x = 1; x < m; x++) { + for (let y = 1; y < n; y++) { + const node1 = an[x + ioff] + const node2 = bn[y + joff] + + if (al[i] === al[x + ioff] && bl[j] === bl[y + joff]) { + const costs = [ + fd[x - 1][y] + costFn.remove(node1), + fd[x][y - 1] + costFn.insert(node2), + fd[x - 1][y - 1] + costFn.update(node1, node2), + ] + + fd[x][y] = Math.min(...costs) + const minIndex = costs.indexOf(fd[x][y]) + + if (minIndex === 0) { + partialOps[x][y] = [ + ...partialOps[x - 1][y], + { type: 'remove', node: node1 } + ] + } else if (minIndex === 1) { + partialOps[x][y] = [ + ...partialOps[x][y - 1], + { type: 'insert', node: node2 } + ] + } else { + const opType: 'match' | 'update' = fd[x][y] === fd[x - 1][y - 1] ? 'match' : 'update' + partialOps[x][y] = [ + ...partialOps[x - 1][y - 1], + { type: opType, left: node1, right: node2 } + ] + } + + operations[x + ioff][y + joff] = partialOps[x][y] + treeDists[x + ioff][y + joff] = fd[x][y] + } else { + const p = al[x + ioff] - 1 - ioff + const q = bl[y + joff] - 1 - joff + const costs = [ + fd[x - 1][y] + costFn.remove(node1), + fd[x][y - 1] + costFn.insert(node2), + fd[p][q] + treeDists[x + ioff][y + joff] + ] + + fd[x][y] = Math.min(...costs) + const minIndex = costs.indexOf(fd[x][y]) + + if (minIndex === 0) { + partialOps[x][y] = [ + ...partialOps[x - 1][y], + { type: 'remove', node: node1 } + ] + } else if (minIndex === 1) { + partialOps[x][y] = [ + ...partialOps[x][y - 1], + { type: 'insert', node: node2 } + ] + } else { + partialOps[x][y] = [ + ...partialOps[p][q], + ...operations[x + ioff][y + joff] + ] + } + } + } + } + } +} + +interface SymbolNodeValue extends Symbol { + id: number + resources: Resource[] +} + +type ResourceNode = TypedNode +export type SymbolNode = TypedNode +type FileNode = TypedNode<{ id: number; fileName: string }, 'file'> +type RootNode = TypedNode<{ }, 'root'> + +type SymbolGraphNode = + | RootNode + | FileNode + | SymbolNode + | ResourceNode + +const configCache = new Map() +function getConfigStr(r: Resource) { + if (configCache.has(r)) { + return configCache.get(r)! + } + + const str = JSON.stringify(r.config) + configCache.set(r, str) + + return str +} + +function createSimilarityHost() { + const jsonSizeCache = new Map() + const jsonDistCache = new Map>() + + const ldCache = new Map() + function ld(a: string, b: string) { + const k = `${a}:${b}` + const c = ldCache.get(k) + if (c !== undefined) { + return c + } + + const d = levenshteinDistance(a, b) + ldCache.set(k, d) + ldCache.set(`${b}:${a}`, d) + + return d + } + + function getSize(obj: any) { + if (jsonSizeCache.has(obj)) { + return jsonSizeCache.get(obj)! + } + + const r = jsonSize(obj) + jsonSizeCache.set(obj, r) + + return r + } + + function getDist(a: any, b: any) { + const cached = jsonDistCache.get(a)?.get(b) + if (cached !== undefined) { + return cached + } + + const r = jsonDist(a, b) + if (!jsonDistCache.has(a)) { + jsonDistCache.set(a, new Map()) + } + if (!jsonDistCache.has(b)) { + jsonDistCache.set(b, new Map()) + } + jsonDistCache.get(a)!.set(b, r) + jsonDistCache.get(b)!.set(a, r) + + return r + } + + function jsonSize(obj: any): number { + if (typeof obj === 'string') { + return obj.length + } + + if (typeof obj === 'number') { + return String(obj).length + } + + if (typeof obj === 'boolean' || obj === null) { + return 1 + } + + if (typeof obj === 'undefined') { + return 0 + } + + if (Array.isArray(obj)) { + return obj.reduce((a, b) => a + getSize(b), 0) + } + + let s = 0 + for (const [k, v] of Object.entries(obj)) { + s += k.length + getSize(v) + } + + return s + } + + function jsonDist(a: any, b: any): number { + if (typeof a !== typeof b) { + return getSize(a) + getSize(b) + } + + if (a === null || b === null) { + if (a === null && b === null) { + return 0 + } + + return getSize(a) + getSize(b) + } + + if (typeof a === 'string') { + return ld(a, b) + } + + if (typeof a === 'number') { + return ld(String(a), String(b)) + } + + if (typeof a === 'boolean') { + return a === b ? 0 : 1 + } + + if (Array.isArray(a) || Array.isArray(b)) { + if (!(Array.isArray(a) && Array.isArray(b))) { + return getSize(a) + getSize(b) + } + + const m = Math.min(a.length, b.length) + let i = 0 + let s = 0 + for (; i < m; i++) { + s += getDist(a[i], b[i]) + } + + const j = Math.max(a.length, b.length) + const c = a.length > b.length ? a : b + for (; i < j; i++) { + s += getSize(c[j]) + } + + return s + } + + // This is a naive solution. A better solution would find the best matching key (tree edit distance) + let s = 0 + for (const [k, v] of Object.entries(a)) { + if (v === undefined) continue + + const u = b[k] + if (u !== undefined) { + s += getDist(v, u) + } else { + s += k.length + } + } + + for (const [k, u] of Object.entries(b)) { + if (u !== undefined && (!(k in a) || a[k] === undefined)) { + s += k.length + } + } + + return s + } + + return { ld, getSize, getDist } +} + +export type SymbolGraph = ReturnType +export function createSymbolGraph(sourceMap: TerraformSourceMap, resources: Record) { + let idCounter = 0 + let resourceIdCounter = 0 + + const nodes = new Map() + const resourceToNode = new Map() // Top scope + const symbolMap = new Map() + + type Scope = { isNewExpression?: boolean; callSite: number; assignment?: number; namespace?: number[] } + + function createSymbolNode(scope: Scope) { + const id = scope.callSite + const base = sourceMap.symbols[id].name + const nameParts = [base] + if (scope.namespace) { + nameParts.unshift(...scope.namespace.map(n => sourceMap.symbols[n].name)) + } + + const exp = `${scope.isNewExpression ? 'new ' : ''}${nameParts.join('.')}` + const name = scope.assignment !== undefined + ? `${sourceMap.symbols[scope.assignment].name} = ${exp}` + : exp + + return createTypedNode('symbol', { + ...sourceMap.symbols[id], + id: idCounter++, + resources: [], + // FIXME: don't clobber the original name + // Might need to add a separate field + name, + }) + } + + function getSymbolNode(scope: Scope) { + const id = scope.callSite + if (nodes.has(id)) { + return nodes.get(id)! + } + + const node = createSymbolNode(scope) + if (scope.assignment !== undefined) { + symbolMap.set(sourceMap.symbols[scope.assignment], node) + } else { + symbolMap.set(sourceMap.symbols[scope.callSite], node) + } + + nodes.set(id, node) + + return node + } + + function getCallsites(resource: string) { + const scopes = sourceMap.resources[resource]?.scopes + if (!scopes) { + return + } + + return scopes.map(s => sourceMap.symbols[s.callSite]) + } + + for (const [k, v] of Object.entries(sourceMap.resources)) { + if (v.scopes.length === 0) { + // This only happens for generated "export" files + getLogger().warn(`Resource "${k}" has no scopes`) + continue + } + + const [rType, ...rest] = k.split('.', 2) + const rName = rest.join('.') + const config = resources[k] + if (!config) { + throw new Error(`Missing resource in source map: ${k}`) + } + + const r: Resource = { + id: resourceIdCounter++, + type: rType, + subtype: rType === 'synapse_resource' ? config.type : undefined, + name: rName, + config: { ...config, type: undefined }, + scope: [], + fileName: '', + } + + const n = getSymbolNode(v.scopes[0]) + n.value.resources.push(r) + resourceToNode.set(k, n) + } + + const root = createTypedNode('root', {}) + + for (let i = 0; i < sourceMap.symbols.length; i++) { + const node = nodes.get(i) + if (!node || node.value.resources.length === 0) continue + + root.children.push(node as any) + if (!node.parents!.includes(root)) { + node.parents!.push(root) + } + + node.value.resources.sort((a, b) => getConfigStr(a).length - getConfigStr(b).length) + } + + root.children.sort((a, b) => { + const d = (a as SymbolNode).value.fileName.localeCompare((b as SymbolNode).value.fileName) + if (d === 0) { + return (a as SymbolNode).value.name.localeCompare((b as SymbolNode).value.name) + } + return d + }) + + function getSymbols() { + return root.children as SymbolNode[] + } + + function findSymbolFromResourceKey(key: string) { + return resourceToNode.get(key) + } + + function isInternalResource(key: string) { + const type = getResourceType(key) + if (type.kind === 'synapse') { + return true + } + + return false + } + + function getResourceType(key: string): { kind: 'terraform' | 'synapse' | 'custom', name: string } { + const parts = key.split('.') + const type = parts[0] === 'data' ? parts[1] : parts[0] + const csType = type === 'synapse_resource' ? resources[key]?.type : undefined + if (!csType) { + return { kind: 'terraform', name: type } + } + + const userType = (csType === 'Example' || csType === 'ExampleData' || csType === 'Custom' || csType === 'CustomData') + ? resources[key].input.type.split('--').at(-2) + : undefined + + if (!userType) { + return { kind: 'synapse', name: csType } + } + + return { kind: 'custom', name: userType } + } + + function getTestInfo(key: string) { + const config = getConfig(key) + if (!config) { + return + } + + const moduleName = (config as any).module_name + if (!moduleName || typeof moduleName !== 'string') { + return + } + + const parsed = parseModuleName(moduleName) + if (parsed.testSuiteId === undefined) { + return + } + + return { fileName: parsed.fileName, testSuiteId: parsed.testSuiteId } + } + + function hasResource(key: string) { + return resourceToNode.has(key) + } + + function matchSymbolNodes(name: string, fileName?: string) { + const nodes: SymbolNode[] = [] + for (const [k, v] of symbolMap) { + if (k.name === name && (!fileName || k.fileName === fileName)) { + nodes.push(v) + } + } + + return nodes + } + + function getConfig(key: string): object | undefined { + return resources[key] + } + + return { + getConfig, + getSymbols, + hasResource, + findSymbolFromResourceKey, + isInternalResource, + getResourceType, + matchSymbolNodes, + getTestInfo, + + getCallsites, + } +} + +export function createSymbolGraphFromTemplate(template: TfJson) { + const newSourceMap = template['//']?.sourceMap + if (!newSourceMap) { + throw new Error(`No new source map found`) + } + + const newResources: Record = {} + for (const [k, v] of Object.entries(template.resource)) { + for (const [k2, v2] of Object.entries(v)) { + const id = `${k}.${k2}` + newResources[id] = v2 + } + } + + return createSymbolGraph(newSourceMap, newResources) +} + +// Used for combining the most recently compiled template vs. the last deployed one +export type MergedGraph = ReturnType +export function createMergedGraph(graph: SymbolGraph, oldGraph?: SymbolGraph) { + function getSymbol(r: string) { + const symbol = graph.findSymbolFromResourceKey(r) + if (symbol) { + return symbol + } + + const oldSymbol = oldGraph?.findSymbolFromResourceKey(r) + if (!oldSymbol) { + throw new Error(`Missing symbol for resource: ${r}`) + } + + if (oldSymbol.value.id < 1000) { + oldSymbol.value.id += 1000 // XXX: no need to explain + } + + return oldSymbol + } + + function getResourceType(r: string) { + if (graph.hasResource(r)) { + return graph.getResourceType(r) + } + + return oldGraph?.getResourceType(r) + } + + function isInternalResource(r: string) { + if (graph.hasResource(r)) { + return graph.isInternalResource(r) + } + + return oldGraph?.isInternalResource(r) + } + + function isTestResource(r: string) { + const config = graph.getConfig(r) ?? oldGraph?.getConfig(r) + if (!config || !('module_name' in config) || typeof config.module_name !== 'string') { + return + } + + const parsed = parseModuleName(config.module_name) + + return parsed.testSuiteId !== undefined + } + + function hasSymbol(r: string) { + return graph.hasResource(r) || oldGraph?.hasResource(r) + } + + function listSymbols() { + return [ + ...graph.getSymbols(), + ...(oldGraph?.getSymbols() ?? []), + ] + } + + function getCallsites(r: string) { + return graph.getCallsites(r) ?? oldGraph?.getCallsites(r) + } + + function getConfig(r: string) { + return graph.getConfig(r) ?? oldGraph?.getConfig(r) + } + + return { + getConfig, + hasSymbol, + getSymbol, + getResourceType, + isInternalResource, + isTestResource, + listSymbols, + getCallsites, + } +} + + +interface Resource { + readonly id: number + readonly type: string + readonly name: string + readonly config: any + readonly subtype?: string + readonly scope: number[] + readonly fileName: string +} + + +// Goal: minimize # of ops for a given deploy +// Cost is determined by the LD between the last deployed config vs. now + +function getResourceKey(r: Resource) { + const t = r.subtype ? `${r.type}.${r.subtype}` : r.type + + return `${t}.${r.name}` +} + + +function printNode(node: SymbolGraphNode) { + switch (node.type) { + case 'file': + return `[file] ${node.value.fileName}` + case 'symbol': + return `[symbol] ${node.value.name} (id: ${node.value.id}; rc: ${node.value.resources.length})` + case 'resource': + return `[resource] ${getResourceKey(node.value)}` + + case 'root': + return '' + } +} + +function printNodeRecursive(node: SymbolGraphNode, showResources = false, depth = 0, stack: SymbolGraphNode[] = []): string[] { + const lines: string[] = [] + lines.push(`${' '.repeat(depth)}${printNode(node)}`) + stack.push(node) + + for (const c of node.children) { + if (stack.includes(c as SymbolGraphNode)) { + throw new Error(`Cycle detected: ${[...stack, c as SymbolGraphNode].map(n => printNode(n)).join(' --> ')}`) + } + if (!showResources && c.type === 'resource') continue + lines.push(...printNodeRecursive(c as SymbolGraphNode, showResources, depth + 1, stack)) + } + + stack.pop()! + + return lines +} + +// function printTree(g: ReturnType) { +// getLogger().raw(printNodeRecursive(g.root).join('\n')) +// } + + +function isSameType(r1: Resource, r2: Resource) { + return r1.type === r2.type && r1.subtype === r2.subtype +} + +function getTypeKey(r: Resource) { + return r.subtype ? `${r.type}.${r.subtype}` : r.type +} + +function createTokenizer(mappings: Record) { + const trie = createTrie() + for (const [k, v] of Object.entries(mappings)) { + trie.insert(k, v) + } + + function mapString(s: string): string[] { + const r: string[] = [] + let t = trie.createIterator() + let j = 0 + let k = 0 + for (let i = 0; i < s.length; i++) { + const n = t.next(s[i]) + if (n.done) { + i -= j + j = 0 + t = trie.createIterator() + } else if (n.value) { + r.push(s.slice(k, i - j)) + r.push(n.value) + j = 0 + k = i + 1 + t = trie.createIterator() + } else { + j += 1 + } + } + if (r.length === 0) { + return [s] + } + if (k !== s.length) { + r.push(s.slice(k)) + } + return r + } + + return { mapString } +} + +export function normalizeConfigs(config: TfJson) { + const maps = { + data: new Map(), + locals: new Map(), + resources: new Map(), + providers: new Map(), + modules: new Map(), + } + + function getMapping(type: keyof typeof maps, key: string) { + const m = maps[type] + if (m.has(key)) { + return m.get(key)! + } + + const x = `${m.size.toString().padStart(7, '0')}` + m.set(key, x) + + return x + } + + const mappings: Record = {} + for (const [k, v] of Object.entries(config.locals ?? {})) { + const m = getMapping('locals', k) + mappings[`local.${k}`] = `local.${m}` + if (k.startsWith('o_')) { + const id = k.slice(2) + mappings[`"${id}"`] = `"${m}"` + } + } + + for (const [k, v] of Object.entries(config.data ?? {})) { + for (const [k2, _] of Object.entries(v)) { + const n = `${k}.${k2}` + const m = getMapping('data', n) + mappings[`data.${n}`] = `data.${m}` + } + } + + for (const [k, v] of Object.entries(config.resource ?? {})) { + for (const [k2, c] of Object.entries(v)) { + if (c.module_name && typeof c.module_name === 'string') { + const [name, fragment] = c.module_name.split('#') + const m = getMapping('modules', name) + c.module_name = fragment ? `${m}#${fragment}` : m + } + const n = `${k}.${k2}` + const m = getMapping('resources', n) + mappings[n] = m + } + } + + for (const [k, v] of Object.entries(config.provider ?? {})) { + for (const p of v) { + const alias = p.alias + if (!alias) continue + + const n = `${k}.${alias}` + const m = getMapping('providers', n) + mappings[n] = m + } + } + + // XXX: this string fragment is very very common + mappings['${jsonencode({"Version" = "2012-10-17", "Statement" = [{"Effect" = "Allow", "Action" = '] = '__IAM_POLICY_FRAGMENT__' + + const tokenizer = createTokenizer(mappings) + + function visit(c: any): any { + if (typeof c === 'string') { + return tokenizer.mapString(c) + } + + if (Array.isArray(c)) { + return c.map(visit) + } + + if (typeof c !== 'object' || !c) { + return c + } + + const o: any = {} + for (const [k, v] of Object.entries(c)) { + o[k] = visit(v) + } + return o + } + + for (const [k, v] of Object.entries(config.resource ?? {})) { + for (const [k2, c] of Object.entries(v)) { + const type = c.type + v[k2] = visit(c) + if (type !== undefined) { + v[k2].type = type + } + } + } + + return config +} + + +function printOp(op: Operation) { + switch (op.type) { + case 'insert': + case 'remove': + return `[${op.type}] ${printNode(op.node)}` + case 'match': + case 'update': + return `[${op.type}] ${printNode(op.left)} -> ${printNode(op.right)}` + } +} + +export function createGraphSolver( + newResources: Record, + newSourceMap: TerraformSourceMap, + oldResources: Record, + oldSourceMap: TerraformSourceMap, + newDeps: Record>, + oldDeps: Record>, +) { + const g1 = createSymbolGraph(newSourceMap, newResources) + const g2 = createSymbolGraph(oldSourceMap, oldResources) + + const { ld, getSize, getDist } = createSimilarityHost() + + const resourceIndex = new Map>() + function getResourceGroups(n: SymbolNode) { + if (resourceIndex.has(n)) { + return resourceIndex.get(n)! + } + + const o: Record = {} + for (const r of n.value.resources ?? []) { + const t = getTypeKey(r) + const a = o[t] ??= [] + a.push(r) + } + resourceIndex.set(n, o) + + return o + } + + + // Assumption: all resources have the same type + function getLowerBoundResources(g1: Resource[], g2: Resource[]) { + const l1 = g1.map(x => getSize(x.config)).sort() + const l2 = g2.map(x => getSize(x.config)).sort() + const m = Math.min(g1.length, g2.length) + let i = 0 + let upper = 0 + let lower = 0 + for (; i < m; i++) { + upper += Math.max(l1[i], l2[i]) + lower += Math.abs(l1[i] - l2[i]) + } + + const rem = (l1.length > l2.length ? l1 : l2).slice(i).reduce((a, b) => a + b, 0) + upper += rem + lower += rem + + return lower / upper + } + + function getLowerBoundResourcesSymbol(a: SymbolNode, b: SymbolNode) { + const g1 = getResourceGroups(a) + const g2 = getResourceGroups(b) + let c = 0 + + for (const [k, v] of Object.entries(g1)) { + const o = g2[k] + if (o === undefined) { + c += v.length + } else { + c += getLowerBoundResources(v, o) + } + } + + for (const [k, v] of Object.entries(g2)) { + if (!(k in g1)) { + c += v.length + } + } + + const lb = c / (a.value.resources.length + b.value.resources.length) + if (lb > 1 || Number.isNaN(lb)) { + throw new Error(`Bad lower bound: ${lb}`) + } + + return lb + } + + + function compareSymbolDeps(a: SymbolNode, b: SymbolNode) { + const g1 = getResourceGroups(a) + const g2 = getResourceGroups(b) + } + + function compareResource(newResource: Resource, oldResource: Resource, excludedNew: Set, excludedOld: Set) { + const nd = newDeps[`${newResource.type}.${newResource.name}`] + const od = oldDeps[`${oldResource.type}.${oldResource.name}`] + } + + function matchResources(oldResources: Resource[], newResources: Resource[]) { + const r1 = oldResources!.map(x => createTypedNode('resource', x)) + const r2 = newResources!.map(x => createTypedNode('resource', x)) + const matches = findSimplePaths( + (u, openSet) => { + const sizeU = getSize(u.value.config) + + let m = Infinity + let o: typeof u | undefined + for (const v of openSet) { + const sizeV = getSize(v.value.config) + const delta = sizeU - sizeV + if (o && Math.abs(delta) >= m) { + // if (delta > 0) { + // break + // } + continue + } + + const d = getDist(u.value.config, v.value.config) + if (d === 0) { + return [d, v] + } + if (d < m) { + m = d + o = v + } + } + + return [m, o] + }, + r1, + r2 + ) + + return matches + } + + + function matchResourcesByGroup(oldSymbol: SymbolNode, newSymbol: SymbolNode) { + const matches: Record = {} + const g1 = getResourceGroups(oldSymbol) + const g2 = getResourceGroups(newSymbol) + + const visited = new Set() + for (const [k, v] of Object.entries(g1)) { + const v2 = g2[k] + if (v2) { + visited.add(k) + matches[k] = matchResources(v, v2).map(x => ({ score: x[0], from: x[1].value, to: x[2].value })) + } + } + + for (const [k, v] of Object.entries(g2)) { + if (visited.has(k)) continue + const v2 = g1[k] + if (v2) { + matches[k] = matchResources(v, v2).map(x => ({ score: x[0], from: x[2].value, to: x[1].value })) + } + } + + return matches + } + + function resourceGroupDist(oldResources: Resource[], newResources: Resource[]) { + const z = matchResources(oldResources, newResources) + + let tc = 0 + const seenOld = new Set() + const seenNew = new Set() + for (const [cc, oldNode, newNode] of z) { + tc += cc + seenOld.add(oldNode.value) + seenNew.add(newNode.value) + } + + const sizeOld = oldResources.length + const sizeNew = newResources.length + if ((sizeNew - seenNew.size) > 0 && seenOld.size !== sizeOld) { + throw new Error('Invariant broken') + } + + if ((sizeOld - seenOld.size) > 0 && seenNew.size !== sizeNew) { + throw new Error('Invariant broken') + } + + const m = Math.min(sizeOld, sizeNew) + if (z.length !== m) { + throw new Error(`Invariant broken: ${z.length} !== ${m}`) + } + + + tc += oldResources.filter(x => !seenOld.has(x)).map(x => getSize(x.config)).reduce((a, b) => a + b, 0) + tc += newResources.filter(x => !seenNew.has(x)).map(x => getSize(x.config)).reduce((a, b) => a + b, 0) + + // Matching the smallest values is treated as the worst-case scenario because + // our goal is to find the best possible matches, _not_ minimize the total cost + const l1 = oldResources.map(x => getSize(x.config)).sort((a, b) => a - b) + const l2 = newResources.map(x => getSize(x.config)).sort((a, b) => a - b) + let i = 0 + let t = 0 + for (; i < m; i++) { + t += Math.max(l1[i], l2[i]) + } + t += (l1.length > l2.length ? l1 : l2).slice(i).reduce((a, b) => a + b, 0) + + const total = tc / t + if (total > 1) { + throw new Error(`Total must be less than 1: ${total} > 1 (${tc} / ${t})`) + } + + return total * (l1.length + l2.length) + } + + function symbolResourceDiff(a: SymbolNode, b: SymbolNode) { + let c = 0 + const g1 = getResourceGroups(a) + const g2 = getResourceGroups(b) + + for (const [k, v] of Object.entries(g1)) { + const v2 = g2[k] + if (v2) { + c += resourceGroupDist(v, v2) + } else { + c += v.length + } + } + + for (const [k, v] of Object.entries(g2)) { + if (!(k in g1)) { + c += v.length + } + } + + // const m = a.value.resources.length + b.value.resources.length + // if (c > m) { + // throw new Error(`Bad total: ${c} [maxLength: ${m}]`) + // } + + return c + } + + const ldn = keyedMemoize((a: string, b: string) => { + const d = ld(a, b) + + return d / Math.max(a.length, b.length) + }) + + function symbolNameDiff(a: SymbolNode, b: SymbolNode) { + const ndn = ldn(a.value.name, b.value.name) + const fdn = ldn(a.value.fileName, b.value.fileName) + + return (ndn / 2) + (fdn / 2) + } + + function groupByFile>(nodes: Iterable) { + const m = new Map() + + for (const n of nodes) { + if (!m.has(n.value.fileName)) { + m.set(n.value.fileName, []) + } + m.get(n.value.fileName)!.push(n) + } + + for (const arr of m.values()) { + arr.sort((a, b) => a.value.name.localeCompare(b.value.name)) + } + + return m + } + + const symbolDiffCache = new Map() + function symbolDiff(a: SymbolNode, b: SymbolNode) { + const k = `${a.value.id}:${b.value.id}` + if (symbolDiffCache.has(k)) { + return symbolDiffCache.get(k)! + } + + const d = symbolResourceDiff(a, b) + const m = a.value.resources.length + b.value.resources.length + if (d > m) { + throw new Error(`Bad math: ${d} > ${m}`) + } + + const c = (((1 + symbolNameDiff(a, b)) * (1 + (d / m))) - 1) / 3 + symbolDiffCache.set(k, c) + + return c + } + + interface Key { + next(m: number, n: number): Key + next(m: number, n: number | undefined): Key + next(m: number | undefined, n: number): Key + toString(): string + } + + function createPosKey(): Key { + const keys: Record = {} + + function inner(posM: number[], posN: number[], str = ''): Key { + function next(m: number | undefined, n: number | undefined) { + const nPosM = m !== undefined ? [...posM, m].sort() : posM + const nPosN = n !== undefined ? [...posN, n].sort() : posN + const k = `${nPosM.join('.')}:${nPosN.join('.')}` + + return keys[k] ??= inner(nPosM, nPosN, k) + } + + function toString() { + return str + } + + return { next, toString } + } + + return inner([], []) + } + + function findSimplePaths>( + getBestDist: (u: T, openSet: Iterable, byFile: T[], minScore?: number) => readonly [number, T | undefined], + oldSymbols: T[], + newSymbols: Iterable + ) { + interface State { + pairs: [cost: number, old: T, new: T][] + rem: T[] + openSet: Set + score: number + fScore: number + key: Key + } + + const pq = createMinHeap((a, b) => { + const d = a.fScore - b.fScore + if (d === 0) { + return a.rem.length - b.rem.length + } + return d + }) + + const openSet = new Set(newSymbols) + const maxMatches = Math.min(oldSymbols.length, openSet.size) + pq.insert({ + pairs: [], + rem: [...oldSymbols], + openSet, + score: 0, + fScore: maxMatches, + key: createPosKey(), + }) + + const scores = new Map() + const byFile = groupByFile(openSet) + + while (true) { + const s = pq.extract() + if (s.pairs.length === maxMatches) { + return s.pairs + } + + const u = s.rem[0] + const bf = byFile.get(u.value.fileName)?.filter(y => s.openSet.has(y)) ?? [] + const r = getBestDist(u, s.openSet, bf) + if (!r[1]) { + throw new Error(`No matches: ${u}`) + } + + const os = new Set(s.openSet) + os.delete(r[1]) + + const rem = s.rem.slice(1) + const ns = s.score + r[0] + 1 + const k = s.key.toString() + const ls = scores.get(k) + if (ls !== undefined && ns >= ls) { + continue + } + + scores.set(k, ns) + + // if (r[0] > 0 && !greedy) { + // lowerBounds![u.value.id] = r[0] + // const fScore = s.score + (maxMatches - s.pairs.length) + r[0] + lb + // pq.insert({ ...s, rem: [...rem, u], fScore }) + // bounds.set(k, lowerBounds!) + // } + + pq.insert({ + rem, + score: ns, + openSet: os, + pairs: [...s.pairs, [r[0], u, r[1]]], + fScore: ns + (maxMatches - (s.pairs.length + 1)), + key: s.key.next(u.value.id, r[1].value.id), + }) + } + } + + function createEstimator(f1: SymbolNode[], f2: Iterable) { + const counts: Record = {} + let total = 0 + for (const f of f1) { + for (const [k, v] of Object.entries(getResourceGroups(f))) { + counts[k] = (counts[k] ?? 0) + v.length + total += v.length + } + } + for (const f of f2) { + for (const [k, v] of Object.entries(getResourceGroups(f))) { + counts[k] = (counts[k] ?? 0) - v.length + total += v.length + } + } + + function next(n: SymbolNode | undefined, o: SymbolNode | undefined) { + const z = { ...counts } + if (n) { + for (const [k, v] of Object.entries(getResourceGroups(n))) { + z[k] = (z[k] ?? 0) - v.length + } + } + if (o) { + for (const [k, v] of Object.entries(getResourceGroups(o))) { + z[k] = (z[k] ?? 0) + v.length + } + } + + return Object.values(z).reduce((a, b) => a + Math.abs(b), 0) + } + + function getTotal() { + return total + } + + return { next, getTotal } + } + + function checkSymbols(symbols: SymbolNode[]) { + for (const s of symbols) { + if (s.value.resources.length === 0) { + throw new Error(`Symbol has no resources: ${printNode(s)}`) + } + } + } + + function findSymbolMatches(oldSymbols: SymbolNode[], newSymbols: SymbolNode[]) { + checkSymbols(oldSymbols) + checkSymbols(newSymbols) + + interface State { + currentNodes?: [SymbolNode, SymbolNode] + ops: Operation[], + rem: SymbolNode[] + openSet: Set + score: number + fScore: number + key: Key + } + + const pq = createMinHeap((a, b) => { + const d = a.fScore - b.fScore + if (d === 0) { + return a.rem.length - b.rem.length + } + return d + }) + + // `u` goes with `oldSymbols` (rem) + + function findPerfectMatches(u: SymbolNode, byFile: SymbolNode[]) { + const matches: [SymbolNode, SymbolNode][] = [] + for (const v of byFile) { + if (v.value.name !== u.value.name) continue + + const lb = getLowerBoundResourcesSymbol(u, v) + if (lb !== 0) continue + + const d = symbolDiff(u, v) + if (d === 0) { + matches.push([u, v]) + } + } + + return matches + } + + function findBestMatches(rem: SymbolNode[], openSet: Iterable) { + const scores: [number, SymbolNode, SymbolNode][] = [] + + for (const u of rem) { + for (const v of openSet) { + const m = scores[v.value.id]?.[0] ?? Infinity + const lb = getLowerBoundResourcesSymbol(u, v) + if (lb === 1 || (lb / 3) > m) { + continue + } + + const nd = symbolNameDiff(u, v) + if ((nd / 3) > m) { + continue + } + + const d = symbolDiff(u, v) + if (d <= m) { + scores[v.value.id] = [d, u, v] + } + } + } + + return scores + } + + function findBestInsertMatches(rem: SymbolNode[], openSet: Iterable) { + const scores: [number, SymbolNode, SymbolNode][] = [] + + for (const v of openSet) { + for (const u of rem) { + const m = scores[u.value.id]?.[0] ?? Infinity + const lb = getLowerBoundResourcesSymbol(u, v) + if (lb === 1 || (lb / 3) > m) { + continue + } + + const nd = symbolNameDiff(u, v) + if ((nd / 3) > m) { + continue + } + + const d = symbolDiff(u, v) + if (d <= m) { + scores[u.value.id] = [d, u, v] + } + } + } + + return scores + } + + function getInitial() { + const rem: SymbolNode[] = [] + const openSet = new Set(newSymbols) + const byFile = groupByFile(openSet) + const ops: Operation[] = [] + + for (const u of oldSymbols) { + const matches = findPerfectMatches(u, byFile.get(u.value.fileName)?.filter(x => openSet.has(x)) ?? []) + if (matches.length === 0) { + rem.push(u) + continue + } + + if (matches.length > 1) { + const first = matches.shift()! + openSet.delete(first[1]) + ops.push({ type: 'match', left: u, right: first[1] }) + // getLogger().log('perfect match (dupe)', printNode(u), printNode(first[1])) + continue + } + + const [_, v] = matches[0] + openSet.delete(v) + ops.push({ type: 'match', left: u, right: v }) + // getLogger().log('perfect match', u.value.id, v.value.id) + } + + const imbalance = rem.length - openSet.size + + if (imbalance > 0) { + const w = findBestMatches(rem, openSet).filter(x => x !== undefined).sort((a, b) => b[0] - a[0]) + const missing = rem.filter(x => !w.find(y => y[1] === x)) + + function remove(n: SymbolNode) { + const idx = rem.indexOf(n) + ops.push({ type: 'remove', node: rem.splice(idx, 1)[0] }) + } + + for (let i = 0; i < imbalance; i++) { + if (missing.length > 0) { + remove(missing.shift()!) + } else { + const m = w.shift()! + remove(m[1]) + } + } + } + + if (imbalance < 0) { + const w = findBestInsertMatches(rem, openSet).filter(x => x !== undefined).sort((a, b) => b[0] - a[0]) + const missing = [...openSet].filter(x => !w.find(y => y[2] === x)) + + function insert(n: SymbolNode) { + ops.push({ type: 'insert', node: n }) + openSet.delete(n) + } + + for (let i = 0; i < Math.abs(imbalance); i++) { + if (missing.length > 0) { + insert(missing.shift()!) + } else { + const m = w.shift()! + insert(m[2]) + } + } + } + + return { + ops, + rem, + openSet, + score: 0, + fScore: rem.length, + key: createPosKey(), + } + } + + const scores = new Map() + const initial = getInitial() + pq.insert(initial) + + while (true) { + const s = pq.extract() + const opsLeft = s.rem.length + if (opsLeft === 0) { + return s.ops + } + + // getLogger().log(s.ops.length - initial.ops.length, s.score, s.fScore, pq.length) + + if (s.currentNodes) { + const [u, v] = s.currentNodes + + const d = symbolDiff(u, v) + + const ns = s.score + d + 1 + const nextKey = s.key.next(u.value.id, v.value.id) + const k = nextKey.toString() + const ls = scores.get(k) + if (ls === undefined || ns < ls) { + scores.set(k, ns) + + const rem = s.rem.slice(1) + const openSet = new Set(s.openSet) + openSet.delete(v) + + // Not entirely accurate + const isMatch = ( + u.value.name === v.value.name && + u.value.fileName === v.value.fileName && + u.value.resources.length === v.value.resources.length + ) + + pq.insert({ + rem, + ops: [...s.ops, { + type: !isMatch ? 'update' : 'match', + left: u, + right: v + }], + openSet, + score: ns, + fScore: ns + (opsLeft - 1) * 2, // We defer expanding paired nodes + key: nextKey, + }) + } + + continue + } + + const u = s.rem[0] + const g1 = getResourceGroups(u) + const c: Record = {} + for (const [k, v] of Object.entries(g1)) { + c[k] = v.length + } + + const est = createEstimator(s.rem, s.openSet) + + for (const v of s.openSet) { + const g2 = getResourceGroups(v) + const cd = { ...c } + for (const [k, v] of Object.entries(g2)) { + cd[k] = (cd[k] ?? 0) - v.length + } + + const m = u.value.resources.length + v.value.resources.length + const rh = Object.values(cd).reduce((a, b) => a + Math.abs(b), 0) / m + if (rh !== 1) { + const nd = symbolNameDiff(u, v) + const h2 = (((1 + nd) * (1 + rh)) - 1) / 3 + const h3 = est.next(u, v) / ((est.getTotal() - m) || 1) + + pq.insert({ + ...s, + currentNodes: [u, v], + fScore: s.score + (h2 + (h3 / 3) + opsLeft), + }) + } + } + } + } + + function solve() { + return findSymbolMatches(g2.getSymbols(), g1.getSymbols()) + } + + return { + solve, + matchResourcesByGroup, + } +} + +export function detectRefactors( + newResources: Record, + newSourceMap: TerraformSourceMap, + oldResources: Record, + oldSourceMap: TerraformSourceMap, + newDeps: Record>, + oldDeps: Record>, +) { + const moves = new Map() + const gs = createGraphSolver(newResources, newSourceMap, oldResources, oldSourceMap, newDeps, oldDeps) + const ops = gs.solve() + + for (const op of ops) { + if (op.type === 'match' || op.type === 'update') { + const m = gs.matchResourcesByGroup(op.left, op.right) + for (const g of Object.values(m)) { + for (const m of g) { + const from = `${m.from.type}.${m.from.name}` + const to = `${m.to.type}.${m.to.name}` + + if (from !== to) { + const p = moves.get(from)?.to + if (p !== undefined && p !== to) { + throw new Error(`Detected conflict for resource "${from}": ${p} !== ${to} [${op.type} ${printNode(op.left)}, ${printNode(op.right)}]`) + } + moves.set(from, { + to, + fromSymbol: op.left.value, + toSymbol: op.right.value, + }) + } + } + } + } else { + getLogger().log('Extra op', printOp(op)) + } + } + + return [...moves.entries()].map(([k, v]) => ({ from: k, to: v.to, fromSymbol: v.fromSymbol, toSymbol: v.toSymbol })) +} + +function mapScope(scope: { callSite: number; assignment?: number; namespace?: number[] }, sourcemap: TerraformSourceMap) { + return { + callSite: sourcemap.symbols[scope.callSite], + assignment: scope.assignment ? sourcemap.symbols[scope.assignment] : undefined, + namespace: scope.namespace ? scope.namespace.map(x => sourcemap.symbols[x]) : undefined, + } +} + +function mapScopes(sourcemap: TerraformSourceMap) { + return Object.fromEntries(Object.entries(sourcemap.resources).map(([k, v]) => [k, v.scopes.map(x => mapScope(x, sourcemap))])) +} + + +// A "resource symbol" is composed of multiple identifiers that form +// the origin point for one or more resources. +interface ResourceSymbol { + readonly id: number + readonly pos: number + readonly name: string + readonly fileName: string + readonly resources: Resource[] + readonly namespace?: { readonly name: string; readonly pos: number }[] + readonly assignment?: { readonly name: string; readonly pos: number } +} + +// function getResourceSymbols(): ResourceSymbol[] { +// return [] +// } + +export function renderSymbolLocation(sym: Pick, includePosition = false) { + const pos = `:${sym.line + 1}:${sym.column + 1}` + + return `${sym.fileName}${includePosition ? pos : ''}` +} + +export function renderSymbol(sym: Symbol, includeFileName = true, includePosition = false) { + const suffix = includeFileName ? ` ${renderSymbolLocation(sym, includePosition)}` : '' + + return `${sym.name}${suffix}` +} + +// TODO: +// Implement heuristics for detecting "splits" and "merges" +// +// Example of a "split": +// +// --- OLD CODE --- +// +// function foo() { +// const x = new Bar() +// // do stuff with `x` +// } +// +// --- NEW CODE --- +// +// const x = new Bar() +// foo(x) +// +// function foo(x: Bar) { +// // do stuff with `x` +// } +// +// ---------------- +// +// The above is a very common type of refactor when +// wanting to share code. In this case we wanted to make +// `foo` more modular by extracting out an instantiation +// +// +// +// Example of a "merge": +// +// --- OLD CODE --- +// +// const x = new Bar() +// const y = new Baz(x) +// const z = foobar(x, y) +// +// --- NEW CODE --- +// +// function createFoobar() { +// const x = new Bar() +// const y = new Baz(x) +// +// return foobar(x, y) +// } +// +// const z = createFoobar() +// +// ---------------- +// +// This type of refactor is commonly used to hide intermediate +// steps that were needed to create what the caller wanted +// \ No newline at end of file diff --git a/src/repl.ts b/src/repl.ts new file mode 100644 index 0000000..991d4d2 --- /dev/null +++ b/src/repl.ts @@ -0,0 +1,258 @@ +import * as repl from 'node:repl' +import * as net from 'node:net' +import * as path from 'node:path' +import * as fs from 'node:fs/promises' +import { CombinedOptions } from '.' +import { getLogger } from './logging' +import { getSocketsDirectory, getTargetDeploymentIdOrThrow, getUserSynapseDirectory } from './workspaces' +import { TfState } from './deploy/state' +import { SessionContext } from './deploy/deployment' +import { pointerPrefix } from './build-fs/pointers' +import { getDisplay } from './cli/ui' +import { getArtifactFs } from './artifacts' + +export interface ReplOptions extends CombinedOptions { + onInit?: (instance: ReplInstance) => void +} + +interface ReplInstance { + context: any + setValue(name: string, value: any): void +} + +export async function createReplServer( + target: string, + loader: ReturnType, + options: ReplOptions, +) { + const socketsDir = getSocketsDirectory() + const targetName = target.startsWith(pointerPrefix) + ? target.slice(pointerPrefix.length) + : getTargetDeploymentIdOrThrow() + + const socketAddress = path.resolve(socketsDir, targetName) + if (socketAddress.length > 103) { + throw new Error(`Socket address is too long`) + } + + await fs.mkdir(path.dirname(socketAddress), { recursive: true }) + + const server = net.createServer({}) + await new Promise((resolve, reject) => { + server.listen(socketAddress, () => { + server.removeListener('error', reject) + resolve() + }) + server.once('error', reject) + }) + + getLogger().log('started server at', socketAddress) + server.on('close', () => { + getLogger().log('stopped server at', socketAddress) + }) + + let socket: net.Socket + const dimensions = { columns: 100, rows: 12 } + function resize(columns: number, rows: number) { + dimensions.columns = columns + dimensions.rows = rows + socket?.emit('resize') + } + + server.once('connection', async s => { + getLogger().log('got connection') + s.once('close', () => { + server.close() + instance.close() + }) + + socket = s + + Object.defineProperties(s, { + columns: { get: () => dimensions.columns, enumerable: true, configurable: true }, + rows: { get: () => dimensions.rows, enumerable: true, configurable: true }, + }) + + const instance = await createRepl(target, loader, options, s).catch(e => { + getLogger().error('Failed to start REPL instance', (e as any).message) + s.end() + + throw e + }) + instance.on('close', () => s.end()) + }) + + return { + resize, + address: socketAddress, + } +} + +async function createRepl( + target: string | undefined, + loader: ReturnType, + options: ReplOptions, + socket?: net.Socket +) { + const [targetModule, loggingModule] = await Promise.all([ + target ? loader.loadModule(target) : undefined, + undefined + ]) + + const instance = repl.start({ + useGlobal: true, + useColors: true, + replMode: repl.REPL_MODE_STRICT, + breakEvalOnSigint: true, // note: does not work w/ custom `eval`, + input: socket, + output: socket, + terminal: socket ? true : undefined, + }) + + // TODO: history should be per-program + const historyFile = path.resolve(getUserSynapseDirectory(), 'repl-history') + instance.setupHistory(historyFile, err => { + if (err) { + getLogger().error(`Failed to setup REPL history`, err) + } + }) + + const names: string[] = [] + + // TODO: make this do more than clearing the screen + instance.defineCommand('reset', () => { + instance.output.write(`\x1b[2J${instance.getPrompt()}`) + }) + + instance.defineCommand('list', () => { + instance.output.write(names.join('\n') + '\n') + instance.output.write(instance.getPrompt()) + }) + + function setValue(name: string, value: any) { + names.push(name) + Object.defineProperty(instance.context, name, { + value, + enumerable: true, + configurable: false, + }) + } + + function initContext() { + if (targetModule) { + for (const [k, v] of Object.entries(targetModule)) { + if (k === '__esModule') continue + + setValue(k, v) + } + } + + options?.onInit?.({ context: instance.context, setValue }) + } + + instance.on('reset', initContext) + // instance.on('close', async () => (await logSubscription)?.dispose()) + + try { + initContext() + } catch (e) { + instance.close() + throw e + } + + return instance +} + +export async function enterRepl( + target: string | undefined, + loader: ReturnType, + options: CombinedOptions +) { + await getDisplay().releaseTty() + const instance = await createRepl(target, loader, options) + + return { + promise: new Promise((resolve, reject) => { + instance.on('exit', resolve) + }) + } +} + +// Looks like this: `0:foo;4:bar;21:qaz` +function parseSymbolBindings(bindings: string): Record { + const parsed: Record = {} + for (const b of bindings.split(';')) { + const [id, name] = parseBinding(b) + parsed[id] = name + } + + return parsed + + function parseBinding(binding: string): [id: string, name: string] { + const [id, name] = binding.split(':', 2) + if (!name) { + throw new Error(`Bad symbol binding: ${binding}`) + } + + return [id, name] + } +} + +export async function getSymbolDataResourceId(resources: Record, fileName: string, executionScope: string) { + const synapseResources: Record = resources?.synapse_resource ?? {} + const symbolDataResourceName = Object.entries(synapseResources).find(([k, v]) => { + return ( + v.type === 'Closure' && + v.input.source === fileName && + v.input.options?.id === '__exported-symbol-data' && + v.input.options?.executionScope === executionScope + ) + })?.[0] + + if (!symbolDataResourceName) { + throw new Error(`No exported symbols found for scope: ${executionScope}`) + } + + return { + resourceId: `synapse_resource.${symbolDataResourceName}` + } +} + +export async function prepareReplWithSymbols(bindings: string, dataResourceId: string, state: TfState) { + const parsedBindings = parseSymbolBindings(bindings) + const [type, ...rest] = dataResourceId.split('.') + const name = rest.join('.') + const symbolDataResource = state.resources + .filter(r => r.type === type && r.name === name) + .map(r => r.state) + .map(r => r.attributes.output.value.destination) + .pop() + + if (!symbolDataResource) { + throw new Error('No exported symbol resource found!') + } + + const artifactFs = await getArtifactFs() + const location = await artifactFs.resolveArtifact(symbolDataResource) + const artifactName = path.basename(location) + + const onInit: ReplOptions['onInit'] = instance => { + const symbols = instance.context.symbols + if (!symbols) { + throw new Error(`Missing symbols object`) + } + + for (const [id, name] of Object.entries(parsedBindings)) { + if (!(id in symbols)) { + throw new Error(`Symbol value not found in execution scope`) + } + + instance.setValue(name, symbols[id]) + } + } + + return { + location: `${pointerPrefix}${artifactName}`, + onInit, + } +} \ No newline at end of file diff --git a/src/runtime/env.ts b/src/runtime/env.ts new file mode 100644 index 0000000..aad84d7 --- /dev/null +++ b/src/runtime/env.ts @@ -0,0 +1,40 @@ +import * as path from 'node:path' +import { getWorkingDir } from '../workspaces' +import { getBuildTarget } from '../execution' + +// Maybe do backticks too +function unquote(str: string) { + const isSingleQuoted = str[0] === "'" && str.at(-1) === "'" + const isDoubleQuoted = !isSingleQuoted && str[0] === '"' && str.at(-1) === '"' + if (isSingleQuoted || isDoubleQuoted) { + return str.slice(1, -1) + } + + return str +} + +export function parseEnvFile(text: string) { + const result: Record = {} + + const lines = text.split(/\r?\n/) + for (const l of lines) { + const sep = l.indexOf('=') + if (sep === -1) { + // bad parse + continue + } + + const key = l.slice(0, sep).trimEnd() + const value = l.slice(sep + 1).trimStart() + result[key] = unquote(value) + } + + return result +} + +export function getCurrentEnvFilePath() { + const environment = getBuildTarget()?.environmentName + const suffix = environment ? `.${environment}` : '' + + return path.resolve(getWorkingDir(), `.env${suffix}`) +} diff --git a/src/runtime/importMaps.ts b/src/runtime/importMaps.ts new file mode 100644 index 0000000..ee21a8a --- /dev/null +++ b/src/runtime/importMaps.ts @@ -0,0 +1,279 @@ +import { getLogger } from '../logging' +import { Mutable } from '../utils' +import { PackageInfo } from './modules/serdes' + +function _showImportMap(m: ImportMap, printSourceInfo?: (source: T) => string) { + function render(map: ImportMap, depth = 0) { + for (const [k, v] of Object.entries(map)) { + const source = v.source ? printSourceInfo?.(v.source) : undefined + getLogger().log(`${' '.repeat(depth)}${depth ? '|__ ' : ''}${k}${source ? ` - ${source}` : ''}`) + if (v.mapping) { + render(v.mapping, depth + 1) + } + } + } + render(m) +} + + +export function showImportMap(m: ImportMap) { + _showImportMap(m, printSource) +} + +function printSource(info: SourceInfo) { + const type = info.type + switch (type) { + case 'package': + return info.data.version + case 'artifact': + return info.data.metadataHash + } + + throw new Error(`Unsupported source type: ${type}`) +} + +export type SourceInfo = { + readonly type: 'package' + readonly data: PackageInfo +} | { + readonly type: 'artifact' + readonly data: { hash: string; metadataHash: string } +} + +export interface ImportMap { + [specifier: string]: { + readonly source?: T + readonly mapping?: ImportMap + readonly location: string + readonly locationType?: 'module' | 'package' + } +} + +function _hoistImportMap(mapping: ImportMap, getKey: (source: T) => string, stack: ImportMap[] = []): ImportMap { + // Import maps can include cycles which we can simply ignore + // + // Hoisting is applied bottom-up, guaranteeing we'll visit any + // detected cycles after visiting all non-cyclical dependencies + + const shouldSkip = new Set>() + for (const [k, v] of Object.entries(mapping)) { + if (!v.mapping) continue + + if (stack.includes(v.mapping)) { + shouldSkip.add(v.mapping) + } else { + _hoistImportMap(v.mapping, getKey, [...stack, v.mapping]) + } + } + + const roots = new Set(Object.keys(mapping)) + const candidates = new Map[]>() + + for (const [k, v] of Object.entries(mapping)) { + if (!v.mapping || shouldSkip.has(v.mapping)) continue + + for (const spec of Object.keys(v.mapping)) { + if (roots.has(spec)) { + continue + } + + const l = candidates.get(spec) ?? [] + l.push(v.mapping) + candidates.set(spec, l) + } + } + + for (const [k, v] of candidates.entries()) { + const groups: Record[]> = {} + for (const m of v) { + const source = m[k].source + if (source) { + const g = groups[getKey(source)] ??= [] + g.push(m) + } + } + + const bestGroup = Object.entries(groups).sort((a, b) => b[1].length - a[1].length).pop() + if (bestGroup) { + mapping[k] = bestGroup[1][0][k] + for (const g of bestGroup[1]) { + delete g[k] + } + } + } + + return mapping +} + +function getKeyFromSource(source: SourceInfo) { + switch (source.type) { + case 'package': + return `${source.data.name}-${source.data.version}` + case 'artifact': + return `${source.data.hash}-${source.data.metadataHash}` + } +} + +export function hoistImportMap(mapping: ImportMap): ImportMap { + function pruneDuplicates(mapping: ImportMap) { + const roots = new Set(Object.keys(mapping)) + for (const k of roots) { + const v = mapping[k] + if (!v.mapping) continue + pruneDuplicates(v.mapping) + + for (const spec of Object.keys(v.mapping)) { + if (!roots.has(spec)) continue + const s1 = v.mapping[spec].source + const s2 = mapping[spec].source + if (s1 && s2 && getKeyFromSource(s1) === getKeyFromSource(s2)) { + delete v.mapping[spec] + } + } + } + return mapping + } + + return pruneDuplicates(_hoistImportMap(mapping, getKeyFromSource)) +} + +export interface FlatImportMap { + readonly sources: Record // ID -> T + readonly mappings: Record > // ID -> (spec -> ID) + readonly locations: Record file(s) + readonly location: string + readonly locationType?: 'module' | 'package' + }> +} + +function _flattenImportMap(mapping: ImportMap, getKey: (source: T) => string, collapse?: boolean): FlatImportMap { + const sources: FlatImportMap['sources'] = {} + const mappings: FlatImportMap['mappings'] = {} + const locations: FlatImportMap['locations'] = {} + + const unknowns = new Map[string], string>() + const encodedIds = new Map() + + function getUnknownSourceId(n: ImportMap[string]) { + if (unknowns.has(n)) { + return unknowns.get(n)! + } + + const newId = `unknown-${unknowns.size}` + unknowns.set(n, newId) + + return newId + } + + const uniqueIds = new Map() + function getId(n: ImportMap[string]) { + if (!collapse) { + if (uniqueIds.has(n)) { + return uniqueIds.get(n)! + } + uniqueIds.set(n, `${uniqueIds.size+1}`) + return uniqueIds.get(n)! + } + + if (!n.source) { + return getUnknownSourceId(n) + } + + return getKey(n.source) + } + + // Makes the import map smaller on disk + function getEncodedId(n: ImportMap[string]) { + const id = getId(n) + if (encodedIds.has(id)) { + return encodedIds.get(id)! + } + + const encoded = `${encodedIds.size}` + encodedIds.set(id, encoded) + + return encoded + } + + function addMapping(parentId: string, spec: string, id: string) { + const m = mappings[parentId] ??= {} + m[spec] = id + } + + function visit(mapping: ImportMap, currentId: string) { + for (const [k, v] of Object.entries(mapping)) { + const id = getEncodedId(v) + addMapping(currentId, k, id) + + if (!(id in locations)) { + sources[id] = v.source + locations[id] = { location: v.location, locationType: v.locationType } + + if (v.mapping) { + visit(v.mapping, id) + } + } + } + } + + visit(mapping, '#root') + + return { sources, mappings, locations } +} + +function _expandImportMap(mapping: FlatImportMap): ImportMap { + if (Object.keys(mapping.mappings).length === 0) { + return {} + } + + const expanded = new Map[string]>() + + function expand(id: string): ImportMap[string] { + if (expanded.has(id)) { + return expanded.get(id)! + } + + const l = mapping.locations[id] + if (!l) { + throw new Error(`Missing import: ${id}`) + } + + const r: Mutable[string]> = { + location: l.location, + locationType: l.locationType, + source: mapping.sources[id], + } + + expanded.set(id, r) + + const m = mapping.mappings[id] + if (m) { + const inner: ImportMap = r.mapping = {} + for (const [k, v] of Object.entries(m)) { + inner[k] = expand(v) + } + } + + return r + } + + const root = mapping.mappings['#root'] + if (!root) { + throw new Error(`Missing root mapping`) + } + + const res: ImportMap = {} + for (const [k, v] of Object.entries(root)) { + res[k] = expand(v) + } + + return res +} + +export function flattenImportMap(mapping: ImportMap, collapse = true) { + return _flattenImportMap(mapping, getKeyFromSource, collapse) +} + +export function expandImportMap(mapping: FlatImportMap) { + return _expandImportMap(mapping) +} \ No newline at end of file diff --git a/src/runtime/loader.ts b/src/runtime/loader.ts new file mode 100644 index 0000000..333c21e --- /dev/null +++ b/src/runtime/loader.ts @@ -0,0 +1,1740 @@ +import * as path from 'node:path' +import * as vm from 'node:vm' +import { createRequire, isBuiltin, SourceMap, SourceOrigin } from 'node:module' +import { SyncFs } from '../system' +import { findSourceMap, SourceMapV3 } from './sourceMaps' +import { getLogger } from '../logging' +import { ArtifactMetadata, getDataRepository, type Artifact } from '../artifacts' +import { AsyncLocalStorage } from 'node:async_hooks' +import { ModuleResolver, ModuleTypeHint } from './resolver' +import { Auth, createAuth, StoredCredentials } from '../auth' +import { getHash, isNonNullable, memoize, throwIfNotFileNotFoundError } from '../utils' +import { CodeCache, Context, copyGlobalThis } from './utils' +import type { resolveValue } from './modules/serdes' +import { applyPointers, coerceToPointer, DataPointer, getNullHash, isDataPointer, isNullHash, toAbsolute } from '../build-fs/pointers' +import { getWorkingDir } from '../workspaces' +import { getFs, getSelfPathOrThrow, isSelfSea } from '../execution' +import { BackendClient } from './modules/core' +import { createNpmLikeCommandRunner } from '../pm/publish' +import { waitForPromise } from '../zig/util' + +export const pointerPrefix = 'pointer:' +export const synapsePrefix = 'synapse:' +export const providerPrefix = 'synapse-provider:' +const seaAssetPrefix = 'sea-asset:' + +export function createContext( + bt: { deploymentId?: string; programId: string }, + backendClient: BackendClient, + auth: Auth, + console?: typeof globalThis.console +) { + const contextStorage = new AsyncLocalStorage>() + + // XXX: need to embed context in all serialized objects... + // Right now we silently fail if there is no context + function getContext(type: string) { + return contextStorage.getStore()?.[type] ?? [{}] + } + + const ctx = copyGlobalThis() + const globals = ctx.globals as any + globals.__getBackendClient = () => backendClient, + globals.__getArtifactFs = () => { + const afs = contextStorage.getStore()?.['afs'][0] + if (!afs) { + throw new Error(`No artifact fs found`) + } + return afs + } + + globals.__getContext = () => { + return { get: getContext } + } + + globals.__getCredentials = createGetCredentials(auth) + globals.__runCommand = createNpmLikeCommandRunner(getWorkingDir()) + globals.__buildTarget = bt + globals.__waitForPromise = waitForPromise + + if (console) { + globals.console = console + globals.globalThis.console = console + } else { + globals.console = globalThis.console + } + + // XXX: we should only expose symbols that we use instead of the entire registry + globals.Symbol = globalThis.Symbol + + globals.Error.stackTraceLimit = 25 // TODO: make configurable + + function runWithNamedContexts(namedContexts: Record, fn: () => T): T { + return contextStorage.run(namedContexts, fn) + } + + function _registerSourceMapParser(parser: SourceMapParser) { + registerSourceMapParser(parser, globals.Error) + } + + return { + ctx, + getContext, + runWithNamedContexts, + registerSourceMapParser: _registerSourceMapParser, + } +} + +export function createGetCredentials(auth: ReturnType) { + const credentials = new Map() + + async function getCredentials(id?: string) { + const account = id ? await auth.getAccount(id) : await auth.getActiveAccount() + if (!account) { + throw new Error('No such account exists') + } + + if (credentials.has(account.id)) { + const creds = credentials.get(account.id)! + if (creds.expiresAt > Date.now()) { + return creds + } + + credentials.delete(account.id) + } + + const creds = await auth.getCredentials(account) + if (!creds) { + throw new Error('No credentials available') + } + + credentials.set(account.id, creds) + + return creds + } + + return getCredentials +} + +export interface BasicDataRepository { + getDataSync(hash: string): Uint8Array + getDataSync(hash: string, encoding: BufferEncoding): string +} + +interface ModuleLoaderOptions { + readonly env?: Record + readonly codeCache?: CodeCache + readonly useThisContext?: boolean + readonly sourceMapParser?: Pick, 'registerFile' | 'registerDeferredMapping' | 'setAlias'> + readonly workingDirectory?: string + readonly deserializer?: typeof resolveValue + readonly typescriptLoader?: (fileName: string, format?: 'cjs' | 'esm') => string + readonly dataRepository?: BasicDataRepository +} + +function createDefaultDataRepo(fs: Pick, dataDir: string): BasicDataRepository { + function getObjectPath(dataDir: string, hash: string) { + if (hash.startsWith(pointerPrefix)) { + hash = hash.slice(pointerPrefix.length) + } + + const prefix = `${hash[0]}${hash[1]}/${hash[2]}${hash[3]}/${hash.slice(4)}` + + return path.resolve(dataDir, prefix) + } + + function getDataSync(hash: string): Uint8Array + function getDataSync(hash: string, encoding: BufferEncoding): string + function getDataSync(hash: string, encoding?: BufferEncoding) { + if (encoding) { + return fs.readFileSync(getObjectPath(dataDir, hash), encoding) + } + return fs.readFileSync(getObjectPath(dataDir, hash)) + } + + return { getDataSync } +} + +export function hydratePointers(repo: BasicDataRepository, id: DataPointer) { + const { hash, storeHash } = id.resolve() + const data = repo.getDataSync(hash, 'utf-8') + if (storeHash === getNullHash()) { + return data + } + + const store = JSON.parse(repo.getDataSync(storeHash, 'utf-8')) + const m = store.type === 'flat' ? store.artifacts[hash] : undefined + if (!m?.pointers) { + return data + } + + return applyPointers(JSON.parse(data), m.pointers) +} + +const esmErrors = [ + "Cannot use import statement outside a module", + "Unexpected token 'export'", + "Cannot use 'import.meta' outside a module", + // This one has to be at the top-level of course + 'await is only valid in async functions and the top level bodies of modules', +] + +const esbuildEsmErrors = [ + 'Top-level await is currently not supported with the "cjs" output format' +] + +declare module "node:vm" { + interface SourceTextModule { + createCachedData(): Buffer + } +} + +const tscHelperLengths: Record = { + __createBinding: [454], + __exportStar: [183], + __generator: [1804], + __awaiter: [664], + __spreadArray: [360], + __read: [476], + +} + +const useStrict = '"use strict";' +const tscHint = '.defineProperty(exports, "__esModule", { value: true });' + + +function parseTscHeader(text: string) { + let pos = 0 + if (text.startsWith(useStrict)) { + pos += useStrict.length + } + + // note that the module could be using `tslib` for `__exportStar` + let hasExportStar = false + let maybeHasTslib = true + let preambleStart: number | undefined + let preambleEnd: number | undefined + + const len = text.length + while (pos < len) { + if (text[pos] === 'v' && text[pos+1] === 'a' && text[pos+2] === 'r' && text[pos+3] === ' ') { + const current = text.slice(pos+4) + const m = current.match(/^([^\s=]+)(\s*)=/) + if (!m) break // bad parse + + const name = m[1] + if (name === '__exportStar') { + hasExportStar = true + } + + // no whitespace = probably minified + const isProbablyMinified = !m[2] + + const offset = tscHelperLengths[name]?.[0] + if (offset) { + const ws = isProbablyMinified ? 0 : 1 + const varLen = 4 + pos += varLen + m[0].length + ws + offset + maybeHasTslib = false + continue + } + } + + if (text[pos] === 'O' && text[pos+1] === 'b' && text[pos+2] === 'j' && text[pos+3] === 'e' && text[pos+4] === 'c' && text[pos+5] === 't') { + const current = text.slice(pos+6) + if (current.startsWith(tscHint)) { + preambleStart = pos+6+tscHint.length + const maybeEnd = current.slice(tscHint.length).indexOf(';') + if (maybeEnd === -1) break // bad parse + preambleEnd = maybeEnd + preambleStart + pos = preambleEnd + 1 + break + } + } + + pos += 1 + } + + if (!preambleStart || !preambleEnd) { + return + } + + const matches = text.slice(preambleStart, preambleEnd).matchAll(/exports\.([^\s\.]+)\s*=\s*/g) + const exports = [...matches].map(m => m[1]) + + if (maybeHasTslib) { + const rem = text.slice(preambleEnd+2) + if (rem.startsWith('const tslib_1')) { + hasExportStar = true + } + } + + if (hasExportStar) { + const rem = text.slice(preambleEnd+1) + //const end = rem.lastIndexOf('exports.') + //if (end === -1) return // bad parse + + const reexportsMatches = rem.matchAll(/__exportStar\(require\("(.+)"\),\s*exports\);/g) + const reexports = [...reexportsMatches].map(m => m[1]) + + return { exports, reexports } + } + + return { exports, reexports: [] } +} + +// TODO: handle cases where whitespace is minified +const esbuildHintStart = '0 && (module.exports = {' + +function parseCjsExports(text: string) { + // `tsc` can emit this near the top of the file + const parsedTscHints = parseTscHeader(text) + if (parsedTscHints) { + return parsedTscHints + } + + // `esbuild` hint annotation at the bottom + // 0 && (module.exports = { + // foo, + // bar + // }); + + const lastSemicolon = text.lastIndexOf(';') + const secondToLastSemicolon = lastSemicolon !== -1 ? text.slice(0, lastSemicolon).lastIndexOf(';') : undefined + const lastSemicolonParen = secondToLastSemicolon ? text.slice(secondToLastSemicolon + 1).lastIndexOf('});') + secondToLastSemicolon + 1 : -1 + + if (lastSemicolonParen !== -1) { + const matchingBrace = text.lastIndexOf('{') + if (matchingBrace !== -1) { + const s = text.slice(matchingBrace - esbuildHintStart.length + 1, matchingBrace + 1) + if (s === esbuildHintStart) { + const lines = text.slice(matchingBrace + 1, lastSemicolonParen).split('\n') + const exports = lines.map(l => l.trim()).map(l => l.endsWith(',') ? l.slice(0, -1) : l).filter(l => !!l) + // FIXME: this only handles shorthand assignments + return { + exports, + reexports: [], + } + } + } + } +} + +function isPromise(obj: T | Promise): obj is Promise { + return obj instanceof Promise || (typeof obj === 'object' && !!obj && 'then' in obj) +} + +function normalizeNodeSpecifier(spec: string) { + return spec.startsWith('node:') ? spec : `node:${spec}` +} + +export interface CjsModule { + exports: any + evaluated?: boolean + paths?: string[] + evaluate: () => any + script?: vm.Script +} + +export interface Module { + // Three identifiers might seem excessive, but it's necessary + readonly id: string // Virtual location, identifies the code within a graph + readonly name: string // This is what the user sees + readonly fileName?: string // Physical location, identifies the code itself + + readonly typeHint?: ModuleTypeHint + + cjs?: CjsModule + esm?: Promise | vm.Module +} + +interface Loader { + createCjs: (module: Module, opt?: ModuleCreateOptions) => CjsModule + createEsm: (module: Module, opt?: ModuleCreateOptions) => Promise | vm.Module +} + +export interface ModuleCreateOptions { + readonly context?: Context + readonly initializeImportMeta?: (meta: ImportMeta, vmModule: vm.SourceTextModule) => void + readonly importModuleDynamically?: (specifier: string, script: vm.Script | vm.Module, importAttributes: ImportAttributes) => Promise | vm.Module +} + +export type ModuleLinker = ReturnType +export function createModuleLinker(fs: Pick, resolver: ModuleResolver, loader: Loader, ctx?: Context) { + const modules: Record = {} + const invertedMap = new Map() + + const linkedModules: Record> = {} + const evaluatingModules = new Map>() + const moduleDeps = new Map() + + const parseCache = new Map>() + + const isSea = isSelfSea() + + function resolveExports(fileName: string, virtualId: string): Iterable { + if (parseCache.has(virtualId)) { + return parseCache.get(virtualId)! + } + + const keys = new Set([]) + parseCache.set(virtualId, keys) + + const text = fs.readFileSync(fileName, 'utf-8') + const parsed = parseCjsExports(text) + if (!parsed) { + return keys + } + + for (const name of parsed.exports) { + keys.add(name) + } + + for (const spec of parsed.reexports) { + const virtualSpecifier = resolver.resolveVirtual(spec, virtualId) + const fileName = resolver.getFilePath(virtualSpecifier) + const resolved = resolveExports(fileName, virtualSpecifier) + for (const n of resolved) { + keys.add(n) + } + } + + return keys + } + + function moduleLinker(spec: string, referencingModule: vm.Module, extra: { attributes: Partial }) { + const importer = getModuleFromVmObject(referencingModule) + const module = getModule(importer.id, spec) + const m = getEsmFromModule(module, extra.attributes) + + let arr = moduleDeps.get(referencingModule) + if (!arr) { + arr = [] + moduleDeps.set(referencingModule, arr) + } + + if (isPromise(m)) { + m.then(m => { arr.push(m) }) + } else { + arr.push(m) + } + + return m + } + + function linkModule(module: vm.Module) { + if (module.status !== 'unlinked') { + if (module.status === 'linking') { + return linkedModules[module.identifier] + } + return module + } + + return linkedModules[module.identifier] = module.link(moduleLinker).then(() => module) + } + + function evaluateLinked(m: vm.Module) { + if (evaluatingModules.has(m)) { + return evaluatingModules.get(m)! + } + + // We have to pre-emptively call `evaluate` on all SourceTextModule deps + const promises: Promise[] = [] + const deps = moduleDeps.get(m) + if (deps) { + for (const d of deps) { + if (!(d instanceof vm.SourceTextModule)) continue + const p = evaluateModule(d) + if (isPromise(p)) { + promises.push(p) + } + } + } + + if (promises.length > 0) { + const p = Promise.all(promises) + .then(() => evaluateModule(m)) + .finally(() => evaluatingModules.delete(m)) + evaluatingModules.set(m, p) + + return p + } + + const p = evaluateModule(m) + if (isPromise(p)) { + const p2 = p.finally(() => evaluatingModules.delete(m)) + evaluatingModules.set(m, p2) + + return p2 + } + + return p + } + + function evaluateEsm(m: vm.Module): Promise | vm.Module { + const linked = linkModule(m) + if (isPromise(linked)) { + return linked.then(evaluateLinked) + } + + return evaluateLinked(linked) + } + + function esmToCjs(esm: vm.Module | Promise) { + let didSetExports = false + const cjs: CjsModule = { + exports: {}, + paths: [], + evaluate: () => { + if (cjs.evaluated) { + return cjs.exports + } + + cjs.evaluated = true + const linked = waitForPromise(isPromise(esm) ? esm.then(linkModule) : linkModule(esm)) + setExports(linked) + waitForPromise(evaluateLinked(linked)) + + return cjs.exports + }, + } + + function setExports(esm: vm.Module) { + if (didSetExports) return + + didSetExports = true + + const names = Object.getOwnPropertyNames(esm.namespace) + for (const k of names) { + Object.defineProperty(cjs.exports, k, { + get: () => (esm.namespace as any)[k], + enumerable: true, + }) + } + + if (!names.includes('__esModule')) { + Object.defineProperty(cjs.exports, '__esModule', { + value: true, + enumerable: true, + }) + } + } + + if (!isPromise(esm) && (esm.status === 'linked' || esm.status === 'evaluated' || esm.status === 'evaluating')) { + setExports(esm) + } + + return cjs + } + + // cjs to esm is slow because you need to know named bindings prior + // to execution (i.e. linking). If the module is guaranteed not to + // depend on any es modules then it's safe to simply execute it + // and inspect the exports to determine named bindings (if any). + // Otherwise, we have to determine the bindings via parsing. + + function cjsToEsm(module: Module, cjs: CjsModule) { + if (module.typeHint === 'builtin') { + const exports = cjs.evaluate() + + return new vm.SyntheticModule([...Object.keys(exports), 'default'], function () { + this.setExport('default', exports.default ?? exports) + for (const [k, v] of Object.entries(exports)) { + this.setExport(k, v) + } + }, { identifier: module.name, context: ctx?.vm }) + } + + const fileName = module.fileName + if (!fileName) { + throw new Error(`Module is missing filename: ${module.id}`) + } + + const keys = [...resolveExports(fileName, module.id)] + const useDefault = keys.length === 0 + const hasDefault = !useDefault && keys.includes('default') + if (!hasDefault) { + keys.push('default') + } + + const m = new vm.SyntheticModule(keys, function () { + const exports = cjs.evaluate() + if (useDefault) { + return this.setExport('default', exports) + } + + const isObject = typeof exports === 'object' && !!exports + if (!isObject) { + throw new Error(`CJS module "${fileName}" did not export an object`) + } + + for (const k of keys) { + if (k !== 'default') { + this.setExport(k, exports[k]) + continue + } + + if (k in exports) { + this.setExport(k, exports[k]) + } else { + this.setExport(k, exports) + } + } + }, { identifier: fileName, context: ctx?.vm }) + + return m + } + + function getTypeHint(spec: string, fileName: string): ModuleTypeHint | undefined { + if (isDataPointer(fileName)) { + return 'pointer' + } + + switch (path.extname(fileName)) { + case '.mjs': + case '.mts': + return 'esm' + case '.cts': + case '.cjs': + return 'cjs' + case '.json': + return 'json' + case '.node': + return 'native' + } + + if (spec.startsWith(pointerPrefix)) { + return 'pointer' + } + } + + function getModule(importer: string, spec: string): Module { + if (isBuiltin(spec)) { + spec = normalizeNodeSpecifier(spec) + if (modules[spec]) { + return modules[spec] + } + + return modules[spec] = { + id: spec, + name: spec, + typeHint: 'builtin', + } + } + + if (isSea && spec.startsWith(seaAssetPrefix)) { + if (modules[spec]) { + return modules[spec] + } + + return modules[spec] = { + id: spec, + name: spec, + typeHint: 'sea-asset', + } + } + + const resolved = resolver.resolveVirtualWithHint(spec, importer) + const hasTypeHint = typeof resolved !== 'string' + const id = hasTypeHint ? resolved[0] : resolved + const fileName = resolver.getFilePath(id) + if (modules[id]) { + return modules[id] + } + + const typeHint = !hasTypeHint ? getTypeHint(spec, fileName) : resolved[1] + + return modules[id] = { + id, + name: spec, // XXX: spec as name is not right + fileName, + typeHint, + } + } + + function createCjs(module: Module) { + const cjs = loader.createCjs(module, { + context: ctx, + importModuleDynamically, + }) + + if (cjs.script) { + invertedMap.set(cjs.script, module) + } + + return cjs + } + + function createEsm(module: Module) { + const esm = loader.createEsm(module, { + context: ctx, + initializeImportMeta, + importModuleDynamically, + }) + + if (isPromise(esm)) { + return esm.then(m => { + invertedMap.set(m, module) + return m + }) + } + + invertedMap.set(esm, module) + return esm + } + + function getCjsFromModule(module: Module) { + if (module.cjs) { + return module.cjs + } + + if (module.typeHint === 'esm' && !module.esm) { + module.esm = createEsm(module) + } + + if (module.esm) { + return module.cjs = esmToCjs(module.esm) + } + + try { + return module.cjs = createCjs(module) + } catch (e) { + if (!(e instanceof ESMError)) { + throw e + } + + module.esm = createEsm(module) + return module.cjs = esmToCjs(module.esm) + } + } + + function getCjs(importer: string, spec: string) { + const module = getModule(importer, spec) + + return getCjsFromModule(module) + } + + function getEsm(importer: string, spec: string, attr?: Record) { + const module = getModule(importer, spec) + + return getEsmFromModule(module, attr) + } + + function createEsmFromModule(module: Module): Promise | vm.Module { + if (module.typeHint === 'builtin') { + module.cjs = createCjs(module) + const exports = module.cjs.evaluate() + + return new vm.SyntheticModule([...Object.keys(exports), 'default'], function () { + this.setExport('default', exports.default ?? exports) + for (const [k, v] of Object.entries(exports)) { + this.setExport(k, v) + } + }, { identifier: module.name, context: ctx?.vm }) + } + + if (module.typeHint === 'pointer') { + return createEsm(module) + } + + if (!module.fileName) { + throw new Error(`Only built-in modules can be missing a filename`) + } + + const fileName = module.fileName + const extname = path.extname(fileName) + switch (extname) { + case '.json': + case '.node': + case '.cjs': { + module.cjs = createCjs(module) + + return cjsToEsm(module, module.cjs) + } + } + + return createEsm(module) + } + + function getEsmFromModule(module: Module, attr?: Record): Promise | vm.Module { + if (module.esm) { + return module.esm + } + + if (module.typeHint === 'cjs' && !module.cjs) { + // XXX: FIXME: we shouldn't need to do this + // it'd be more efficient to determine the module type via the importing pkg + try { + module.cjs = createCjs(module) + } catch (e) { + if (!(e instanceof ESMError)) { + throw e + } + } + } + + if (module.cjs) { + if (module.cjs.evaluated && !parseCache.has(module.id)) { + // We can skip parsing by inspecting the exports directly + const keys = typeof module.cjs.exports === 'object' && !!module.cjs.exports // && cjsModule.exports['__esModule'] + ? Object.getOwnPropertyNames(module.cjs.exports) + : [] + + parseCache.set(module.id, new Set(keys)) + } + + return module.esm = cjsToEsm(module, module.cjs) + } + + return module.esm = createEsmFromModule(module) + } + + function importModule(importer: string, spec: string, attr?: Record): Promise | vm.Module { + const module = getModule(importer, spec) + const esm = isPromise(module) ? module.then(m => getEsmFromModule(m, attr)) : getEsmFromModule(module, attr) + const linked = isPromise(esm) ? esm.then(m => linkModule(m)) : linkModule(esm) + + return isPromise(linked) ? linked.then(evaluateEsm) : evaluateEsm(linked) + } + + function importModuleDynamically(spec: string, referrer: vm.Script | vm.Module, attr: Partial): Promise | vm.Module { + const module = getModuleFromVmObject(referrer) + + return importModule(module.id, spec, attr) + } + + function getModuleFromVmObject(obj: vm.Module | vm.Script) { + const m = invertedMap.get(obj) + if (!m) { + const ident = obj instanceof vm.Module ? obj.identifier : `[Script]` + throw new Error(`Missing module: ${ident}`) + } + return m + } + + function initializeImportMeta(meta: ImportMeta, vmModule: vm.SourceTextModule): void { + const module = getModuleFromVmObject(vmModule) + const fileName = module.fileName! + meta.filename = fileName + meta.dirname = path.dirname(fileName) + meta.url = `file://${fileName}` // probably not valid on Windows + meta.resolve = (spec) => `file://${resolver.resolve(spec, module.id)}` + + // https://github.com/oven-sh/bun/issues/4667 + ;(meta as any).env = process.env + + // Needed for synthesis + ;(meta as any).__virtualId = module.id + } + + function deleteCacheEntry(importer: string, spec: string) { + if (isBuiltin(spec)) { + return + } + + const resolved = resolver.resolveVirtual(spec, importer) + const fileName = resolver.getFilePath(resolved) + + return delete modules[fileName] + } + + function getModuleFromLocation(location: string) { + return modules[location] + } + + return { + getModuleFromLocation, + deleteCacheEntry, + getModule, + getEsm, + getCjs, + linkModule, + + evaluateEsm, + } +} + +function evaluateModule(module: vm.Module) { + if (module.status === 'evaluated' || module.status === 'evaluating') { + return module + } else if (module.status === 'linked') { + return module.evaluate().then(() => module) + } else if (module.status === 'errored') { + throw module.error + } else { + throw new Error(`Bad module state: ${module.status}`) + } +} + +function readSeaAsset(hash: string) { + const sea = require('node:sea') as typeof import('node:sea') + const data = sea.getRawAsset(hash) + if (typeof data === 'string') { + return data + } + + return Buffer.from(data).toString() +} + +export function createModuleLoader( + fs: Pick, + dataDir: string, + resolver: ModuleResolver, + options: ModuleLoaderOptions = {} +) { + const { + env, + codeCache, + sourceMapParser, + deserializer, + workingDirectory = process.cwd(), + dataRepository = createDefaultDataRepo(fs, dataDir), + useThisContext = false, + } = options + + const getDefaultContext = memoize(() => useThisContext ? undefined : copyGlobalThis()) + const isSea = isSelfSea() + + if (useThisContext && typeof WebAssembly === 'undefined') { + const ctx = vm.createContext() + const globals = vm.runInContext('this', ctx) + globalThis.global.WebAssembly = globals.WebAssembly + } + + const getPointerData = (pointer: DataPointer) => { + const data = hydratePointers(dataRepository, pointer) + + // XXX: pretty hacky + if (typeof data === 'string' && data[0] !== '{') { + getLogger().debug(`Treating module "${pointer.hash}" as text`) + + return data + } + + const artifact = typeof data === 'string' ? JSON.parse(data) : data as Artifact + if (artifact.kind === 'compiled-chunk') { + return Buffer.from(artifact.runtime, 'base64').toString('utf-8') + } else if (artifact.kind === 'deployed') { + if (artifact.rendered) { + return Buffer.from(artifact.rendered, 'base64').toString('utf-8') + } + + return artifact + } + + throw new Error(`Unknown artifact kind: ${(artifact as any).kind}`) + } + + const loadPointerData = (artifact: any, importer: string, ctx?: Context, linker = getLinker(ctx)) => { + if (!deserializer) { + throw new Error(`Missing deserializer`) + } + + return deserializer(artifact.captured, { + loadModule: _createRequire(importer, ctx, linker), + }, artifact.table, ctx?.globals) + } + + const nodeCreateRequire = createRequire + const dummyEntrypoint = path.resolve(workingDirectory, '#bootstrap.js') + const defaultRequire = nodeCreateRequire(dummyEntrypoint) + + function requireNodeModule(id: string, require: NodeRequire = defaultRequire) { + switch (id) { + case 'node:os': + return wrapOs(require(id)) + case 'node:path': + return wrapPath(require(id)) + case 'node:fs/promises': + return wrapFsPromises(require(id), dataRepository) + case 'node:child_process': + return wrapChildProcess(require(id)) + } + + return require(id) + } + + function createEsm(m: Module, opt?: ModuleCreateOptions) { + let data: string + + const specifier = m.name + const ctx = opt?.context ?? getDefaultContext() + if (m.typeHint === 'pointer') { + const pointer = coerceToPointer(!isDataPointer(specifier) && m.fileName?.startsWith(pointerPrefix) ? m.fileName : specifier) + const name = toAbsolute(pointer) + + sourceMapParser?.registerDeferredMapping(name, () => { + const { hash, storeHash } = pointer.resolve() + + return getArtifactSourceMap(dataRepository, hash, storeHash) + }) + + const data = getPointerData(pointer) + + if (typeof data !== 'string') { + return new vm.SyntheticModule(Object.keys(data.captured), function () { + const loaded = loadPointerData(data, m.id, ctx) + for (const [k, v] of Object.entries(loaded)) { + this.setExport(k, v) + } + }, { identifier: name, context: ctx?.vm }) + } + + return new vm.SourceTextModule(data, { + context: ctx?.vm, + identifier: name, + importModuleDynamically: opt?.importModuleDynamically as any, + initializeImportMeta: opt?.initializeImportMeta, + }) + } + + switch (path.extname(m.fileName!)) { + case '.ts': + case '.mts': + // TODO: loader shouldn't load read the file + if (options.typescriptLoader) { + data = options.typescriptLoader(m.fileName!, 'esm') + break + } + default: + data = fs.readFileSync(m.fileName!, 'utf-8') + } + + return new vm.SourceTextModule(data, { + identifier: m.fileName!, // this is used for caching. If it changes, make sure to update the caching logic too + context: opt?.context?.vm ?? getDefaultContext()?.vm, + importModuleDynamically: opt?.importModuleDynamically as any, + initializeImportMeta: opt?.initializeImportMeta, + }) + } + + function createCjs(m: Module, opt?: ModuleCreateOptions) { + if (m.typeHint === 'builtin') { + return createSyntheticCjsModule(() => requireNodeModule(m.name)) + } + + const ctx = opt?.context ?? getDefaultContext() + + function createScript(text: string, name: string, cacheKey?: string) { + return createScriptModule( + ctx?.vm, + text, + name, + _createRequire(m.id, ctx), + codeCache, + cacheKey, + sourceMapParser, + wrappedProcess, + id => _createRequire(m.id, ctx)(id), + opt?.importModuleDynamically, + ) + } + + const specifier = m.name + if (m.typeHint === 'pointer') { + const pointer = coerceToPointer(!isDataPointer(specifier) && m.fileName?.startsWith(pointerPrefix) ? m.fileName : specifier) + const name = toAbsolute(pointer) + + sourceMapParser?.registerDeferredMapping(name, () => { + const { hash, storeHash } = pointer.resolve() + + return getArtifactSourceMap(dataRepository, hash, storeHash) + }) + + const data = getPointerData(pointer) + + if (typeof data !== 'string') { + return createSyntheticCjsModule(() => loadPointerData(data, m.id, ctx)) + } + + return createScript(data, name, pointer.hash) + } + + if (isSea && m.typeHint === 'sea-asset') { + const hash = m.name.slice(seaAssetPrefix.length) + const data = readSeaAsset(hash) + + return createScript(data, m.name, hash) + } + + const fileName = m.fileName + if (!fileName) { + throw new Error(`No file name: ${m.name} [${m.id}]`) + } + + const extname = path.extname(fileName) + switch (extname) { + case '.ts': + case '.cts': + if (options.typescriptLoader) { + try { + return createScript(options.typescriptLoader(fileName, 'cjs'), fileName) + } catch (e) { + const m = (e as any).message + if (extname === '.cts' || !esbuildEsmErrors.find(x => m.includes(x))) { + throw e + } + + throw new ESMError(m) + } + } + case '.mjs': + throw new Error(`Not expected: ${m.id}`) + case '.json': + return createSyntheticCjsModule(() => JSON.parse(fs.readFileSync(fileName, 'utf-8'))) + case '.node': + return createSyntheticCjsModule(() => defaultRequire(fileName)) // XXX: FIXME: probably wrong + } + + const contents = fs.readFileSync(fileName, 'utf-8') + const patchFn = resolver.getPatchFn(fileName) + const data = patchFn?.(contents) ?? contents + + return createScript(data, fileName) + } + + // `undefined` = this context + // not great, I know + const linkers = new Map>() + function getLinker(ctx: Context | undefined) { + if (linkers.has(ctx)) { + return linkers.get(ctx)! + } + + const linker = createModuleLinker(fs, resolver, { createCjs, createEsm }, ctx) + linkers.set(ctx, linker) + + return linker + } + + function createRequireCacheProxy(importer: string, linker: ReturnType) { + return new Proxy({}, { + get: (_, spec) => { + if (typeof spec === 'symbol') { + return + } + + const module = linker.getModule(importer, spec) + if (module.cjs && module.cjs.evaluated) { + return module.cjs.exports + } + }, + deleteProperty: (_, spec) => { + if (typeof spec === 'symbol') { + return false + } + + return linker.deleteCacheEntry(importer, spec) ?? false + } + }) + } + + // Used to make sure `cwd()` works as expected. + // Probably doesn't work with the working dir in `path.resolve` + const wrappedProcess = wrapProcess(process, workingDirectory, env) + function _createRequire(location = dummyEntrypoint, ctx: Context | undefined = getDefaultContext(), linker = getLinker(ctx)) { + const isVirtualImporter = location.startsWith(pointerPrefix) + const resolveLocation = isVirtualImporter ? dummyEntrypoint : path.resolve(workingDirectory, location) + const nodeRequire = nodeCreateRequire(resolveLocation) + const cacheProxy = createRequireCacheProxy(location, linker) + + // cjs to esm is slow because you need to know named bindings prior + // to execution (i.e. linking). If the module is guaranteed not to + // depend on any es modules then it's safe to simply execute it + // and inspect the exports to determine named bindings (if any). + // Otherwise, we have to determine the bindings via parsing. + + function require(id: string): any { + const cjs = linker.getCjs(isVirtualImporter ? location : resolveLocation, id) + + return cjs.evaluate() + } + + function resolve(id: string, opt?: { paths?: string[] }) { + if (opt?.paths) { + return nodeRequire.resolve(id, opt) + } + + return resolver.resolve(id, location) + } + + Object.defineProperty(require, 'main', { + get: () => linker.getModuleFromLocation(location)?.cjs + }) + + return Object.assign(require, { + resolve, + cache: cacheProxy, + }) + } + + async function loadEsm(id: string, location = dummyEntrypoint) { + const ctx = getDefaultContext() + const linker = getLinker(ctx) + const importer = location.startsWith(pointerPrefix) ? location : path.resolve(workingDirectory, location) + const esm = await linker.getEsm(importer, id) + await linker.linkModule(esm) + + return esm.evaluate() + } + + // XXX: too lazy to update dependencies + return Object.assign(_createRequire, { + loadEsm + }) +} + +export function createSyntheticCjsModule(evaluate: (exports: any) => any) { + const cjs: CjsModule = { + exports: {}, + evaluate: () => { + if (cjs.evaluated) { + return cjs.exports + } + + cjs.evaluated = true + cjs.exports = evaluate(cjs.exports) + return cjs.exports + } + } + + return cjs +} + +function wrapCode(text: string, params: string[]) { + const sanitized = text.startsWith('#!') + ? text.replace(/^#!.*/, '') + : text + + return ` +(function (${params.join(',')}) { +${sanitized} +})(${params.join(',')}) +`.trim() +} + +const minScriptLengthForCaching = 1_000 + +class ESMError extends Error {} + +export function createScriptModule( + ctx: vm.Context | undefined, + text: string, + location: string, + requireFn: (id: string) => any, + cache?: CodeCache, + cacheKey?: string, + sourceMapParser?: Pick, + _process?: typeof process, + dynamicImporter?: (id: string) => Promise, + moduleImporter?: ModuleCreateOptions['importModuleDynamically'], + moduleObj?: { exports: any }, +): CjsModule { + // v8 validates code cache data using the filename + // if the names don't match, the data is rejected + // The version of the embedder and v8 also matters + const key = text.length >= minScriptLengthForCaching + ? cacheKey ?? getHash(text) + : undefined + + if (key && sourceMapParser) { + sourceMapParser.setAlias(location, key) + } + + const cachedData = key ? cache?.getCachedData(key) : undefined + + const dynamicImport = (id: string) => { + if (!dynamicImporter) { + throw new Error('No dynamic import callback registered') + } + + return dynamicImporter(id) + } + + const produceCachedData = !!cache && !!key && !cachedData + + function evaluate() { + if (module.evaluated) { + return module.exports + } + + module.evaluated = true + if (!ctx) { + Object.assign(globalThis.global, ext) + s.runInThisContext() + } else { + Object.assign(ctx, ext) + s.runInContext(ctx) + } + + if (cachedData && s.cachedDataRejected) { + getLogger().debug(`Rejected cached data`, location) + cache!.evictCachedData(key!) + } else if (produceCachedData) { + cache!.setCachedData(key, s.createCachedData()) + } + + return module.exports + } + + if (moduleObj) { + (moduleObj as any).evaluate = evaluate + } + + const module: CjsModule = (moduleObj as any) ?? { + exports: {}, + evaluate, + } + + const ext = { + require: requireFn, + __filename: location, + __dirname: path.dirname(location), + module, + exports: module.exports, + + process: _process, + dynamicImport, + + // XXX: not a good impl. + eval: (str: string) => { + // TODO: wrap str? + const script = new vm.Script(str, { + importModuleDynamically: moduleImporter as any, + }) + + if (!ctx) { + Object.assign(globalThis.global, ext) + return script.runInThisContext() + } else { + Object.assign(ctx, ext) + return script.runInContext(ctx) + } + }, + } + + function compileScript() { + try { + return new vm.Script(wrapCode(text, ['require', '__filename', '__dirname', 'module', 'exports']), { + filename: sourceMapParser && key ? key : location, + cachedData, + lineOffset: -1, // The wrapped code has 1 extra line before and after the original code + produceCachedData, + importModuleDynamically: moduleImporter as any, + }) + } catch (e) { + if (e && (e as any)?.name === 'SyntaxError' && esmErrors.includes((e as any).message)) { + throw new ESMError((e as any).message) // XXX: why not just compile the script earlier instead of wrapping? + } + throw e + } + } + + sourceMapParser?.registerFile(location, text) + const s = compileScript() + module.script = s + + return module +} + +export function isSourceOrigin(o: SourceOrigin | {}): o is SourceOrigin { + return Object.keys(o).length !== 0 +} + +export function getArtifactSourceMap(repo: BasicDataRepository, hash: string, storeHash: string) { + if (isNullHash(storeHash)) { + return false + } + + try { + const manifest = JSON.parse(repo.getDataSync(storeHash, 'utf-8')) as { artifacts: Record } + const m = manifest.artifacts[hash] + if (!m) { + throw new Error(`Missing metadata`) + } + + const runtimeSourceMap = m?.sourcemaps?.runtime + if (!runtimeSourceMap) { + return false // Sourcemaps were intentionally excluded. Hide any traces. + } + + const [prefix, _, mapHash] = runtimeSourceMap.split(':') + const data = JSON.parse(repo.getDataSync(mapHash, 'utf-8')) + + return data as SourceMapV3 + } catch (e){ + getLogger().warn(`Failed to get sourcemap for object: ${hash}`, e) + + return false + } +} + + +export function getArtifactOriginalLocation(pointer: string | DataPointer, type: 'runtime' | 'infra' = 'runtime') { + if (!isDataPointer(pointer)) { + return pointer + } + + const repo = getDataRepository(getFs()) + const { hash, storeHash } = pointer.resolve() + const metadata = repo.getMetadata(hash, storeHash) + const m = metadata.sourcemaps?.[type] + if (!m) { + return m + } + + const [_prefix, _storeHash, mapHash] = m.split(':') + const data = JSON.parse(Buffer.from(repo.readDataSync(mapHash)).toString('utf-8')) as Required + + // Guesses the start line by skipping over empty lines + let line = 0 + while (line < data.mappings.length && data.mappings[line] === ';') { line += 1 } + + return getOriginalLocation(new SourceMap(data), pointer, line) +} + +function getOriginalLocation(sourcemap: SourceMap, fileName: string, line = 0, column = 0, workingDirectory = getWorkingDir()) { + const entry = sourcemap.findEntry(line, column) + if (!entry.originalSource) { + return `${fileName}:${line + 1}${column + 1}` + } + + const dir = fileName.startsWith(pointerPrefix) ? workingDirectory : path.dirname(fileName) + const source = path.resolve(dir, entry.originalSource) + + return `${source}:${entry.originalLine + 1}:${entry.originalColumn + 1}` +} + +// Needed to prune traces from the SEA build +const selfPath = __filename + +export type SourceMapParser = ReturnType +export function createSourceMapParser( + fs: SyncFs, + resolver?: ModuleResolver, + workingDirectory = process.cwd(), + excludedDirs?: string[] // Excludes any frames from these directories +) { + const deferredSourceMaps = new Map SourceMapV3 | false | undefined>() + const sourceMaps = new Map() + const files = new Map() + const isDebugMode = !!process.env['SYNAPSE_DEBUG'] + const aliases = new Map() + const internalModules = new Map() + + // TODO: remove "AsyncLocalStorage.run (node:async_hooks" call sites if they appear above an internal module + // TODO: remove usercode callsites that are from `__scope__` + // TODO: remove the mapping for the wrapped user module (it's always at the last non-whitespace character) + + function registerMapping(fileName: string, sourcemap: SourceMapV3) { + if ((sourcemap as any).__internal) { + internalModules.set(fileName, sourcemap.sources[0]) + } + + // The node types are wrong here + const mapping = new SourceMap(sourcemap as Required) + sourceMaps.set(fileName, mapping) + + return mapping + } + + function setAlias(original: string, alias: string) { + aliases.set(alias, original) + } + + function resolveAlias(fileName: string) { + if (aliases.has(fileName)) { + return resolveAlias(aliases.get(fileName)!) + } + return fileName + } + + function registerDeferredMapping(fileName: string, fn: () => SourceMapV3 | false | undefined) { + deferredSourceMaps.set(fileName, fn) + } + + function loadFile(fileName: string) { + if (isBuiltinLike(fileName) || fileName.startsWith(pointerPrefix)) { + return + } + + try { + const data = fs.readFileSync(fileName, 'utf-8') + files.set(fileName, data) + + return data + } catch (e) { + throwIfNotFileNotFoundError(e) + + getLogger().debug(`Missing file: ${fileName}`, (e as any).message) + } + } + + function isBuiltinLike(fileName: string) { + return isBuiltin(fileName) || fileName.startsWith('node:internal/') + } + + const isSea = isSelfSea() + + function tryParseSourcemap(fileName: string, shouldLoadFile = false) { + fileName = resolveAlias(fileName) + + if (sourceMaps.has(fileName)) { + return sourceMaps.get(fileName) + } + + if (deferredSourceMaps.has(fileName)) { + const data = deferredSourceMaps.get(fileName)!() + if (!data) { + sourceMaps.set(fileName, data) + return data + } + + return registerMapping(fileName, data) + } + + const physicalLocation = resolver?.getFilePath(fileName) ?? fileName + if (physicalLocation === selfPath && isSea) { + sourceMaps.set(fileName, isDebugMode ? undefined : false) + return isDebugMode ? undefined : false + } + + // Dead code + const sourceInfoNode = resolver?.getSource(fileName) + const sourceInfo = sourceInfoNode?.source + if (sourceInfo?.type === 'artifact') { + const dataDir = path.dirname(path.dirname(path.dirname(physicalLocation))) // XXX: need to do this better + const hash = path.relative(dataDir, physicalLocation).split(path.sep).join('') + const mapping = getArtifactSourceMap(createDefaultDataRepo(fs, dataDir), hash, sourceInfo.data.metadataHash) + + if (!mapping) { + sourceMaps.set(fileName, mapping) + return mapping + } + + return registerMapping(fileName, mapping) + } + + const contents = files.get(physicalLocation) ?? (shouldLoadFile ? loadFile(physicalLocation) : undefined) + if (!contents) { + sourceMaps.set(fileName, undefined) + return + } + + const sourcemap = findSourceMap(physicalLocation, contents) + if (!sourcemap) { + sourceMaps.set(fileName, undefined) + return + } + + if (!(sourcemap instanceof URL)) { + return registerMapping(fileName, sourcemap) + } + + // TODO: should we always swallow errors from bad sourcemaps? + try { + const data = JSON.parse(fs.readFileSync(sourcemap.pathname, 'utf-8')) + + return registerMapping(fileName, data) + } catch (e) { + throwIfNotFileNotFoundError(e) + + getLogger().debug(`Missing source map: ${sourcemap.pathname}`) + sourceMaps.set(fileName, undefined) + } + } + + function registerFile(fileName: string, contents: string) { + files.set(fileName, contents) + } + + function shouldHideFrame(fileName: string) { + fileName = resolveAlias(fileName) + + if (isDebugMode) { + return false + } + + if (internalModules.has(fileName)) { + return true + } + + if (isBuiltinLike(fileName)) { + return true + } + + if (excludedDirs && !!excludedDirs.find(d => fileName.startsWith(d))) { + return true + } + + return false + } + + function prepareStackTrace(err: Error, stackTraces: NodeJS.CallSite[]) { + const stack: string[] = [] + for (let i = 0; i < stackTraces.length; i++) { + const trace = stackTraces[i] + const fileName = trace.getFileName() + if (!fileName) { + stack.push(trace.toString()) + continue + } + + const sourcemap = tryParseSourcemap(fileName, true) + if (!sourcemap) { + if (sourcemap !== false && !shouldHideFrame(fileName)) { + if (aliases.has(fileName)) { + stack.push(trace.toString().replace(fileName, resolveAlias(fileName))) + } else { + stack.push(trace.toString()) + } + } + continue + } + + const line = trace.getLineNumber()! + const column = trace.getColumnNumber()! + const entry = sourcemap.findEntry(line - 1, column - 1) + if (entry.originalLine === undefined) { + stack.push(trace.toString()) + continue + } + + + const dir = deferredSourceMaps.has(resolveAlias(fileName)) ? workingDirectory : path.dirname(resolveAlias(fileName)) // FIXME: `dirname` is probably wrong here + // const source = path.resolve(dir, entry.fileName) + // const originalLocation = `${source}:${entry.lineNumber}:${entry.columnNumber}` + const source = path.resolve(dir, entry.originalSource) + // Makes the traces more compact if the file is within the current dir + const relSource = source.startsWith(workingDirectory) ? path.relative(workingDirectory, source) : source + + const originalLocation = `${relSource}:${entry.originalLine + 1}:${entry.originalColumn + 1}` + const mapped = trace.toString().replace(`${fileName}:${line}:${column}`, originalLocation) + // TODO: detect duplicate frames and omit them. These are often from inlined help functions e.g. `__scope__` + stack.push(mapped) + } + + return [ + `${err.name}: ${err.message}`, + ...stack.map(l => ` at ${l}`) + ].join('\n') + } + + return { + setAlias, + registerFile, + registerMapping, + tryParseSourcemap, + registerDeferredMapping, + prepareStackTrace, + } +} + +export function registerSourceMapParser(parser: SourceMapParser, errorClass: typeof Error): { dispose: () => void } { + const originalFn = errorClass.prepareStackTrace + errorClass.prepareStackTrace = parser.prepareStackTrace + + function dispose() { + if (errorClass.prepareStackTrace === parser.prepareStackTrace) { + errorClass.prepareStackTrace = originalFn + } + } + + return { dispose } +} + +function createModuleWrap(mod: any, overrides: Record) { + let trappedProto = false + const keys = new Set(Object.keys(overrides)) + const p: any = new Proxy(mod, { + get: (target, prop, recv) => { + if (typeof prop === 'string' && keys.has(prop)) { + return overrides[prop as keyof typeof overrides] + } + + return Reflect.get(target, prop, recv) + }, + + // TODO: we should only trap the prototype if we're being treated as an ES module e.g. `__esModule` was accessed + getPrototypeOf: () => !trappedProto ? (trappedProto = true, p) : null, + }) + + return p +} + +function wrapPath(path: typeof import('node:path')): typeof import('node:path') { + const overrides = { + basename: (p: string) => { + if (isDataPointer(p)) { + return p.hash + } + return path.basename(p) + } + } + + return createModuleWrap(path, overrides) +} + +function wrapFsPromises(fs: typeof import('node:fs/promises'), repo: BasicDataRepository): typeof import('node:fs/promises') { + const overrides = { + readFile: async (fileName: string, opt?: BufferEncoding | any) => { + if (isDataPointer(fileName)) { + const encoding = typeof opt === 'string' ? opt : opt?.encoding + return repo.getDataSync(fileName.hash, encoding as any) + } + return fs.readFile(fileName, opt) + } + } + + return createModuleWrap(fs, overrides) +} + +function wrapChildProcess(child_process: typeof import('node:child_process')): typeof import('node:child_process') { + const _fork = child_process.fork + child_process.fork = function fork(modulePath: string, argsOrOpt?: string[] | any, opt?: any) { + if (isDataPointer(modulePath)) { + return _fork(toAbsolute(modulePath), argsOrOpt, opt) + } + + return _fork(modulePath, argsOrOpt, opt) + } + + return child_process +} + +function wrapOs(os: typeof import('node:os')): typeof import('node:os') { + // Don't wrap if there's no overrides + if (!process.env['NODE_ARCH']) { + return os + } + + const overrides = { + arch: () => process.env['NODE_ARCH'] || os.arch(), + platform: () => process.env['NODE_PLATFORM'] || os.platform(), + endianness: () => process.env['NODE_ENDIANNESS'] || os.endianness(), + } + + return createModuleWrap(os, overrides) +} + +function wrapProcess(proc: typeof process, workingDirectory: string, env?: Record): typeof process { + const mergedEnv = { ...proc.env, ...env } + const arch = mergedEnv['NODE_ARCH'] || proc.arch + const platform = mergedEnv['NODE_PLATFORM'] || proc.platform + + return new Proxy(proc, { + get: (target, prop, recv) => { + if (prop === 'cwd') { + return () => workingDirectory + } else if (prop === 'arch') { + return arch + } else if (prop === 'platform') { + return platform + } else if (prop === 'env') { + return mergedEnv + } + + return Reflect.get(target, prop, recv) + }, + }) +} \ No newline at end of file diff --git a/src/runtime/modules/core.ts b/src/runtime/modules/core.ts new file mode 100644 index 0000000..797a3c7 --- /dev/null +++ b/src/runtime/modules/core.ts @@ -0,0 +1,983 @@ +//# moduleId = synapse:core + +import * as terraform from 'synapse:terraform' + +// This is duplicated from `synapse:terraform` +/** @internal */ +export interface Symbol { + name: string + line: number // 0-indexed + column: number // 0-indexed + fileName: string +} + +/** @internal */ +export interface Scope { + name?: string + moduleId?: string + contexts?: any[] + namespace?: Symbol[] // This is only relevant for property accesses + isNewExpression?: boolean + isStandardResource?: boolean + + // Used for mapping resource instantiations to source code + symbol?: Symbol + + assignmentSymbol?: Symbol +} + +declare function __addTarget(...args: any[]): void +declare function __getCurrentId(): string +declare function __getPermissions(target: any): any +declare function __getContext(): { run(scope: Scope, fn: (...args: any[]) => any, ...args: any[]): any, get: (type: string) => any } +declare function __getBuildDirectory(): string +declare function __getConsole(id?: string, name?: string): any +declare function __getBackendClient(): BackendClient +declare function __requireSecret(envVar: string, type: string): void +declare function __getArtifactFs(): ArtifactFs +declare function __cwd(): string +declare function __waitForPromise(promise: Promise | T): T + +declare function dynamicImport(specifier: string): Promise + +// AUTH +declare function __getCredentials(id?: string): Promise<{ expiresAt: number; access_token: string }> + +// UTIL +declare function __runCommand(cmdOrExecutable: string, args?: string[]): Promise +declare function __createAsset(target: string, importer: string): DataPointer + +interface Logger { + log: (...args: any[]) => void +} + +/** @deprecated */ +declare function __getLogger(): Logger + +// export interface LogEvent { +// readonly timestamp: string | number // ISO8601 or Unix epoch +// readonly data: string | { message: string } | any +// } + +/** @internal */ +export function getCurrentId() { + if (typeof __getCurrentId === 'undefined') { + return '' + } + + return __getCurrentId() +} + +export function runCommand(cmd: string): Promise +export function runCommand(executable: string, args: string[]): Promise +export function runCommand(cmdOrExecutable: string, args?: string[]) { + if (typeof __runCommand === 'undefined') { + throw new Error(`Not implemented outside of Synapse runtime`) + } + + return __runCommand(cmdOrExecutable, args) +} + +/** @internal */ +export function createAsset(target: string, importer: string): DataPointer { + if (typeof __createAsset === 'undefined') { + throw new Error(`Not implemented outside of Synapse runtime`) + } + + return __createAsset(target, importer) +} + +//# resource = true +export function asset(path: string): OpaquePointer { + throw new Error(`Failed to transform "asset" calls`) +} + +/** @internal */ +export function cwd() { + if (typeof __cwd === 'undefined') { + return process.cwd() + } + + return __cwd() +} + +const pointerPrefix = 'pointer:' + +/** @internal */ +export function importArtifact(id: string): Promise { + // A bare hash is OK, metadata may be applied separately + if (typeof id !== 'string' || id.startsWith(pointerPrefix)) { + return dynamicImport(id) + } + + return dynamicImport(`${pointerPrefix}${id}`) +} + +/** @internal */ +export function getCredentials(id?: Identity['id']) { + const envCreds = process.env['COHESIBLE_AUTH'] + if (envCreds) { + return JSON.parse(envCreds) as ReturnType + } + + if (typeof __getCredentials !== 'undefined') { + return __getCredentials(id) + } + + const os = require('node:os') as typeof import('node:os') + const path = require('node:path') as typeof import('node:path') + const fs = require('node:fs/promises') as typeof import('node:fs/promises') + + return (async function () { + const synapseDir = process.env['SYNAPSE_INSTALL'] ?? path.resolve(os.homedir(), '.synapse') + const credsDir = path.resolve(synapseDir, 'credentials') + const statePath = path.resolve(credsDir, 'state.json') + const state = JSON.parse(await fs.readFile(statePath, 'utf-8')) + const target = id ?? state.currentAccount + if (!target) { + throw new Error(`No account selected`) + } + + const creds = JSON.parse(await fs.readFile(path.resolve(credsDir, `${target}.json`), 'utf-8')) + + return creds as ReturnType + })() +} + +function failMissingRuntime(name: string): never { + throw new Error(`Cannot use "${name}" outside of the Synapse runtime`) +} + +/** @internal */ +export function waitForPromise(promise: Promise | T): T { + if (typeof __waitForPromise === 'undefined') { + failMissingRuntime('waitForPromise') + } + return __waitForPromise(promise) +} + +/** @deprecated @internal */ +export function getLogger(): Logger { + if (typeof __getLogger === 'undefined') { + return console + } + + return __getLogger() +} + +// The resource 'backend': +// 1. Logical and phyiscal identifiers +// 2. At least one CRUD operation +// * Without a `Create` operation then resources must be instantiated by reference (i.e. a "data source") + +// More than one get/update operation pairs on a single resource implies that +// the resource is actually composed of more than one resource + +export const context = Symbol.for('context') +export const contextType = Symbol.for('contextType') +const permissions = Symbol.for('permissions') +const moveable2 = Symbol.for('__moveable__2') + +// AWS only +interface Statement { + Effect?: 'Allow' | 'Deny' // Defaults to `Allow` + Action: string | string[] + Resource: string | string[] + Condition?: any + + // Only relevant for managed resources. This field is treated as metadata. + Lifecycle?: LifecycleStage[] +} + +type LifecycleStage = 'create' | 'update' | 'read' | 'delete' + +type Binding = ((this: U, ...args: T) => R) | Statement | Statement[] +interface Context { + // AWS SPECIFIC + partition: string + accountId: string + regionId: string + addStatement(statement: Statement): void + + // GENERIC + createUnknown(): any +} + +type ExtractSignature = T extends { + (...args: infer P): Promise + (...args: infer P2): infer R2 + (...args: infer P3): infer R3 +} ? [P, Partial] : T extends (...args: infer P) => infer R ? [P, Partial] : never + +type Methods = { [P in keyof T]: T[P] extends (...args: any[]) => any ? P : never }[keyof T] +type PermissionsModel = { [P in Methods]+?: Binding[0], ExtractSignature[1], T & { $context: Context }> } +type ConstructorPermissionsModel any> = (this: InstanceType & { $context: Context }, ...args: ConstructorParameters) => InstanceType | void + +/** @internal */ +export function bindModel(ctor: new () => T, model: PermissionsModel): void +export function bindModel(ctor: new (...args: any[]) => T, model: PermissionsModel): void +export function bindModel(ctor: new (...args: any[]) => T, model: PermissionsModel): void { + _bindModel(ctor, model, 'class') +} + +/** @internal */ +export function bindConstructorModel any>(ctor: T, model: ConstructorPermissionsModel): void { + _bindModel(ctor, model, 'constructor') +} + +/** @internal */ +export function bindFunctionModel any>(fn: T, model: Binding, Awaited>, { $context: Context }>): void { + _bindModel(fn, model, 'function') +} + +/** @internal */ +export function bindObjectModel>(obj: T, model: PermissionsModel): void { + _bindModel(obj, model, 'object') +} + +// `Model` is dependent on the target +type Model = any | any[] + +interface ObjectPermissionsBinding { + type: 'object' + methods: Record +} + +interface ClassPermissionsBinding { + type: 'class' + methods: Record + $constructor?: Model +} + +interface FunctionPermissionsBinding { + type: 'function' + call: Model +} + +// Legacy +interface ContainerPermissionsBinding { + type: 'container' + properties: Record +} + +type PermissionsBinding = + | ObjectPermissionsBinding + | ClassPermissionsBinding + | FunctionPermissionsBinding + | ContainerPermissionsBinding + +function mergeBindings(left: PermissionsBinding, right: PermissionsBinding | undefined) { + if (!right) { + return left + } + + if (left.type !== right.type) { + throw new Error(`Cannot merge bindings of different types: ${left.type} !== ${right.type}`) + } + + function mergeObject(a: Record, b: Record) { + const keys = new Set([...Object.keys(a), ...Object.keys(b)]) + const result: Record = {} + + for (const k of keys) { + const l = a[k] + const r = b[k] + + if (!r) { + result[k] = l + } else if (!l) { + result[k] = r + } else if (Array.isArray(l)) { + result[k] = [...l, ...(Array.isArray(r) ? r : [r])] + } else if (Array.isArray(r)) { + result[k] = [...r, ...(Array.isArray(l) ? l : [l])] + } else { + result[k] = [l, r] + } + } + + return result + } + + switch (left.type) { + case 'class': + case 'object': + return { + type: left.type, + methods: mergeObject(left.methods, (right as any).methods), + $constructor: mergeObject( + { $constructor: (left as any).$constructor }, + { $constructor: (right as any).$constructor }, + ).$constructor + } + } + + throw new Error(`Merging not implemented for type: ${left.type}`) +} + +function _bindModel(target: any, model: any, type: 'class' | 'object' | 'function' | 'container' | 'constructor') { + // FIXME: not robust at all + const m = type === 'function' + ? { type, call: model } + : type === 'container' + ? { type, properties: model } + : type === 'constructor' + ? { type: 'class' as const, $constructor: model, methods: {} } + : { type, methods: model } + + target[permissions] = type !== 'container' ? + type !== 'function' + ? mergeBindings(m, target[permissions]) + : m + : { + type: 'container', + properties: { + ...target[permissions]?.properties, + ...model + } + } + + // Bubble up permission models to any parent objects + if (moveable2 in target) { + const operations = target[moveable2]().operations + if (operations.length === 2 && operations[1].type === 'get') { + _bindModel( + target[Symbol.for('unproxyParent')], + { [operations[1].property]: type !== 'function' ? { type, methods: model } : { type, call: model } }, + 'container' + ) + } + } +} + +// Notes: +// * Permissions/network solutions can be asymmetric; the changes needed on the subject may not be the +// same as the changes needed on the actor +// * The above means that we may need to know both the subject and the actor in order to provide a solution +// * Connectivity may not necessarily need to be solved in both directions i.e. it can be one way +// * Rendering models with unknown inputs results in a more permissive solution. The least permissive +// solution can only be found by deferring until final synthesis. + +/** @internal */ +export function getPermissions(target: any): any { + if (typeof __getPermissions === 'undefined') { + return [] + } + + return __getPermissions(target) +} + +declare function __defer(fn: () => void): void + +/** @internal */ +export function getPermissionsLater(target: any, fn: (result: any) => void) { + __defer(() => void fn(getPermissions(target))) +} + +export function defer(fn: () => void) { + __defer(() => void fn()) +} + +interface LocalMetadata { + readonly name?: string + readonly source?: string + readonly publishName?: string + readonly dependencies?: string[] +} + +/** @internal must live in 'core' to be accurate */ +export function peekResourceId>( + target: new (...args: any[]) => T, +): string { + if (!(terraform.peekNameSym in target)) { + throw new Error(`Unable to get resource id from target`) + } + + return (target[terraform.peekNameSym] as any)() +} + +/** @internal */ +export interface ArtifactFs { + writeFile(fileName: string, data: Uint8Array, metadata?: LocalMetadata): Promise + writeFileSync(fileName: string, data: Uint8Array, metadata?: LocalMetadata): DataPointer + writeArtifact(data: Uint8Array, metadata?: LocalMetadata): Promise + writeArtifactSync(data: Uint8Array, metadata?: LocalMetadata): DataPointer + readArtifact(pointer: string): Promise + readArtifactSync(pointer: string): Uint8Array + resolveArtifact(pointer: string, opt?: { name?: string, extname?: string }): Promise +} + +/** @internal */ +export function getArtifactFs(): ArtifactFs { + if (typeof __getArtifactFs === 'undefined') { + throw new Error(`Cannot use artifact fs outside of runtime`) + } + + return __getArtifactFs() +} + +const browserImplSym = Symbol.for('browserImpl') +export function addBrowserImplementation(target: T, alt: U): void { + if (browserImplSym in target) { + throw new Error(`Target function already has a registered browser implementation: ${(target[browserImplSym] as any).name}`) + } + + Object.assign(target, { [browserImplSym]: alt }) +} + +/** @internal */ +export function getBackendClient(): BackendClient { + if (typeof __getBackendClient === 'undefined') { + throw new Error(`Cannot call "getBackendClient" outside of the compiler runtime`) + } + + return __getBackendClient() +} + +interface ContextConstructor { + readonly [contextType]: string + new (...args: any[]): T +} + +export function maybeGetContext(ctor: ContextConstructor): T | undefined { + if (typeof __getContext === 'undefined') { + return + } + + const type = ctor[contextType] + + return __getContext().get(type)?.[0] +} + +export function getContext(ctor: ContextConstructor): T { + if (typeof __getContext === 'undefined') { + return {} as any + } + + // TODO: change how contexts are added and use `at(-1)` instead + const type = ctor[contextType] + const ctx = __getContext().get(type)?.[0] + if (ctx === undefined) { + throw new Error(`Not within context of type "${type}"`) + } + + return ctx +} + +/** @internal */ +export function scope(scope: Scope, fn: (...args: any[]) => any, ...args: any[]): any { + if (typeof __getContext === 'undefined') { + return fn(...args) + } + + return __getContext().run(scope, fn, args) +} + +/** @internal */ +export function getOutputDirectory() { + if (typeof __getBuildDirectory === 'undefined') { + return '' + } + + return __getBuildDirectory() +} + +/** @internal */ +export function getConsole(id?: string, name?: string) { + if (typeof __getConsole === 'undefined') { + return console as any // Type-erased so we don't ref node types + } + + return __getConsole(id, name) +} + +export function addTarget< + T extends abstract new (...args: any[]) => any, + U extends T +>( + base: T, + replacement: U, + targets: 'aws' | 'azure' | 'gcp' | 'local' +): void + +// export function addTarget T>( +// base: (...args: A) => T, +// replacement: U, +// targets: 'aws' | 'azure' | 'gcp' | 'local' +// ): void + +export function addTarget(...args: any[]) { + if (typeof __addTarget === 'undefined') { + return + } + + return __addTarget(...args) +} + +// TODO: should `update` be given the old args in addition to the new args? +// Maybe add it to `this` + +interface ResourceDefinition< + I extends object = object, + T extends I = I, + U extends any[] = [] +> { + read(state: I): T | Promise + create?(...args: U): T | Promise + update?(state: T, ...args: U): T | Promise + delete?(state: T, ...args: U): void | Promise +} + +interface ResourceDefinitionOptionalRead< + T extends object = object, + I extends object = T, + U extends any[] = [] +> { + read?(state: I): T | Promise + create(...args: U): T | Promise + update?(state: T, ...args: U): T | Promise + delete?(state: T, ...args: U): void | Promise +} + +type ResourceConstructor< + I extends object = object, + T extends object = object, + U extends any[] = [], + D extends object = T +> = { + new (...args: U): Readonly + + // The below static method is only safe if the class behaves the same without any initialization logic + // import

any>(this: P, state: I): InstanceType

+} + +export function defineResource< + T extends object = object, + I extends object = T, + U extends any[] = [] +>( + definition: ResourceDefinitionOptionalRead +): ResourceConstructor + +export function defineResource< + I extends object = object, + T extends I = I, + U extends any[] = [] +>( + definition: ResourceDefinition +): ResourceConstructor + +export function defineResource( + definition: ResourceDefinition | ResourceDefinitionOptionalRead +): ResourceConstructor { + if (typeof __getCurrentId === 'undefined' || typeof arguments[1] !== 'string') { + return (class {}) as any + } + + return createCustomResourceClass(arguments[1], definition) +} + +type SerializeableKeys = { [P in keyof T]: T[P] extends (...args: any[]) => any ? never : P }[keyof T] +type Serializeable = Pick> +type Serialized = Readonly>> + +export function using(ctx: T, fn: (ctx: T) => U): U { + if (typeof __getContext === 'undefined') { + return fn(ctx) + } + + return __getContext().run({ contexts: [ctx] }, fn) +} + +/** + * @internal + * + * Binds a secret to a deploy-time environment variable + * + * Currently has no effect at runtime + */ +export function requireSecret(envVar: string, type: string) { + __requireSecret(envVar, type) +} + +// interface LogEvent { +// sessionId: string +// resourceId: string // Terraform logical id for now +// timestamp: string // ISO8601 +// data: T +// } + +/** @internal */ +export interface Secret { + value: string + expiration?: string +} + +/** @internal */ +export interface SecretProvider { + getSecret(): Promise +} + +interface Identity { + readonly id: string + readonly attributes: Record +} + +/** @internal */ +export type AuthenticateFn = (pollToken: string) => Promise +/** @internal */ +export type StartAuthenticationFn = () => Promise<{ pollToken: string, redirectUrl: string }> +/** @internal */ +export interface Provider { + readonly name?: string + readonly type: string + readonly authenticate: AuthenticateFn | { invoke: AuthenticateFn } + readonly startAuthentication: StartAuthenticationFn | { invoke: StartAuthenticationFn } +} +/** @internal */ +export interface Project { + readonly id: string + readonly name?: string + readonly gitRepository?: { readonly url: string } +} +/** @internal */ +export interface SecretsClient { + getSecret(secretType: string): Promise + putSecret(secretType: string, secret: Secret): Promise + deleteSecret(secretType: string): Promise + createSecretProvider(secretType: string, handler: (() => Promise) | { invoke: () => Promise }): Promise + deleteSecretProvider(secretType: string): Promise +} + +/** @internal */ +export interface AuthClient { + createIdentityProvider(idp: Provider): Promise<{ id: string }> + deleteIdentityProvider(id: string): Promise + createMachineIdentity(attributes?: Record): Promise<{ id: string; privateKey: string }> + deleteMachineIdentity(id: string): Promise + getMachineCredentials(id: string, privateKey: string): ReturnType +} + +/** @internal */ +export interface ProjectsClient { + createProject(repo: { name: string; url: string }): Promise + deleteProject(id: Project['id']): Promise +} +/** @internal */ +export interface BackendClient extends SecretsClient, AuthClient, ProjectsClient { + getState(resourceId: string): Promise + getToolDownloadUrl(type: string, opt?: { os?: string; arch?: string; version?: string }): Promise<{ url: string; version: string }> +} +/** @internal */ +export interface ReplacementHook { + beforeDestroy(oldInstance: T): Promise + afterCreate(newInstance: T, state: U): Promise +} + +const pointerSymbol = Symbol.for('synapse.pointer') + +/** @internal */ +export type DataPointer = string & { + readonly ref: string + readonly hash: string; + resolve(): { hash: string; storeHash: string } + isResolved(): boolean + isContainedBy(storeId: string): boolean +} + +/** @internal */ +export function isDataPointer(ref: unknown): ref is DataPointer { + return (typeof ref === 'object' || typeof ref === 'function') && !!ref && (ref as any)[pointerSymbol] +} + +export type OpaquePointer = string & { [pointerSymbol]: unknown } + +// //# resource = true +// export declare class Provider { +// constructor(props?: any) +// } + +const synapseOutput = Symbol.for('synapseClassOutput') + +export function defineDataSource( + handler: (...args: U) => Promise | T, + opt?: { forceRefresh?: boolean } +): (...args: U) => T { + if (typeof __getCurrentId === 'undefined' || typeof arguments[arguments.length - 1] !== 'string') { + return (() => {}) as any + } + + const ds = createCustomResourceClass(arguments[arguments.length - 1], { data: handler }) + + return (...args) => { + const v = ds.import(...args) + if (typeof opt === 'object' && opt?.forceRefresh) { + updateLifecycle(v, { force_refresh: true }) + } + + return (v as any)[synapseOutput] + } +} + +// Common node symbols that are useful in general +// declare global { +// var __filename: string +// var __dirname: string +// } + +interface SynapseProviderProps { + readonly endpoint: string + readonly buildDirectory: string + readonly workingDirectory: string + readonly outputDirectory: string +} + +/** @internal */ +export const Provider = terraform.createSynapseClass('Provider', 'provider') + +interface ObjectDataOutput { + readonly filePath: string +} + +interface ObjectDataInput { + readonly value: any +} + +const ObjectData = terraform.createSynapseClass('ObjectData', 'data-source') + +/** @internal */ +export class SerializedObject extends ObjectData { + constructor(target: any, id?: string) { + // TODO: add a flag/field so we can hide these resources in UI + super({ value: terraform.Fn.serialize(target) }) + + if (id) { + terraform.overrideId(this, id) + } + } +} + +interface ClosureProps { + readonly captured: any + readonly globals?: any + readonly location?: string + readonly options?: any + readonly source?: string +} + +interface ClosureOutput { + readonly destination: string + readonly extname?: string +} + +/** @internal */ +export const Closure = terraform.createSynapseClass('Closure') +/** @internal */ +export const Artifact = terraform.createSynapseClass<{ url: string }, { filePath: string }>('Artifact', 'data-source') + +interface ModuleExportsProps { + readonly source: string + readonly exports: any +} + +const Exported = terraform.createSynapseClass('ModuleExports') + +/** @internal */ +export class ModuleExports extends Exported { + constructor(source: string, exports: any) { + const id = source.replace(/\.(.*)$/, '').replace(/\//g, '--') + + super({ + source, + exports: new SerializedObject(exports, id + '-exports').filePath, + }) + + terraform.overrideId(this, id) + } +} + +interface CustomResourceProps { + readonly type: string + readonly handler: string + readonly plan: any + readonly context?: any +} + +/** @internal */ +export const Custom = terraform.createSynapseClass('Custom') +/** @internal */ +export const CustomData = terraform.createSynapseClass('CustomData', 'data-source') + +class Export extends Closure { + public constructor(id: string, target: any) { + super({ + source: `${id.replace(/--/g, '-')}.ts`, + options: { bundled: false }, + captured: new SerializedObject(target, id + '-captured').filePath, + }) + + terraform.overrideId(this, id) + } +} + +interface AssetProps { + readonly path: string + readonly type?: number + readonly filePath?: string + readonly extname?: string + readonly extraFiles?: Record // dest (relative path/url) -> source +} + +interface AssetOutput { + readonly filePath: string + readonly sourceHash?: string +} + +/** @deprecated @internal */ +export const Asset = terraform.createSynapseClass('Asset') + +/** @internal */ +export class CustomResource extends Custom { + public constructor(type: string, handler: string, ...args: any[]) { + const context = { + 'aws': __getContext().get('aws'), + 'fly-app': __getContext().get('fly-app') // XXX: make this generic + } + + super({ + type, + handler, + plan: new SerializedObject(args).filePath, + context: new SerializedObject(context).filePath, + }) + } +} + +class CustomDataClass extends CustomData { + public constructor(type: string, handler: string, ...args: any[]) { + super({ + type, + handler, + plan: new SerializedObject(args).filePath, + }) + } +} + +const kCustomResource = Symbol.for('customResource') + +function createCustomResourceClass(id: string, definition: any): any { + let def: Export + + // Lazy init + const getDef = () => def ??= new Export(id, definition) + + return class extends CustomResource { + static [kCustomResource] = true + + constructor(...args: any[]) { + super(id, getDef().destination, ...args) + } + + static import(...args: any[]) { + return new CustomDataClass(id, getDef().destination, ...args) + } + } +} + +interface ResourceLifecycle { + create_before_destroy?: boolean + prevent_destroy?: boolean + /** @internal */ + force_refresh?: boolean + ignore_changes?: 'all' | (keyof T)[] + replace_triggered_by?: any[] + /** @internal */ + hook?: { + kind: 'replace' + input: any + handler: string + }[] +} + +// `exclude` is a hack because `getAllResources` is too aggressive +export function updateLifecycle(obj: T, lifecycle: ResourceLifecycle, exclude?: any[]) { + const resolvedLifecycle = { ...lifecycle } + if (lifecycle.replace_triggered_by) { + resolvedLifecycle.replace_triggered_by = terraform.getAllResources(lifecycle.replace_triggered_by, true) + } + + const excluded = exclude?.flatMap(o => terraform.getAllResources(o)) // XXX: this is a big hack + const expandedTarget = terraform.getAllResources(obj).filter(x => !excluded?.includes(x)) + expandedTarget.forEach(t => { + terraform.updateResourceConfiguration(t, o => { + if (!('lifecycle' in o)) { + (o as any).lifecycle = [resolvedLifecycle] + } else { + // TODO: add merge logic + (o as any).lifecycle[0] = { + ...(o as any).lifecycle[0], + ...resolvedLifecycle, + } + } + }) + }) +} + +export function addDependencies(obj: T, ...deps: any[]) { + const expandedTarget = terraform.getAllResources(obj) + const expandedDeps = deps.flatMap(d => terraform.getAllResources(d)) + expandedTarget.forEach(t => { + terraform.updateResourceConfiguration(t, o => { + if (!('depends_on' in o)) { + (o as any).depends_on = [] + } + expandedDeps.forEach(d => { + if (!(o as any).depends_on.includes(d)) { + (o as any).depends_on.push(d) + } + }) + }) + }) +} + +interface ApiRegistrationProps { + readonly kind: string + readonly config: string // pointer +} + +const ApiRegistration = terraform.createSynapseClass('ApiRegistration', 'resource') + +/** @internal */ +export interface SecretProviderProps { + readonly secretType: string + readonly getSecret: () => Promise | Secret +} + +//# resource = true +/** @internal */ +export class SecretProvider2 extends ApiRegistration { + public constructor(props: SecretProviderProps) { + super({ + kind: 'secret-provider', + config: new SerializedObject(props).filePath, + }) + } +} + +interface IdentityProviderProps { + readonly name?: string + readonly type: string + readonly authenticate: AuthenticateFn | { invoke: AuthenticateFn } + readonly startAuthentication: StartAuthenticationFn | { invoke: StartAuthenticationFn } +} + +//# resource = true +/** @internal */ +export class IdentityProvider extends ApiRegistration { + public constructor(props: IdentityProviderProps) { + super({ + kind: 'identity-provider', + config: new SerializedObject(props).filePath, + }) + } +} \ No newline at end of file diff --git a/src/runtime/modules/http.ts b/src/runtime/modules/http.ts new file mode 100644 index 0000000..4171743 --- /dev/null +++ b/src/runtime/modules/http.ts @@ -0,0 +1,975 @@ +//# moduleId = synapse:http +//# transform = persist + +import * as http from 'node:https' +import * as zlib from 'node:zlib' +import * as crypto from 'node:crypto' +import { addBrowserImplementation } from 'synapse:core' + +type TypedArray = + | Uint8Array + | Uint8ClampedArray + | Uint16Array + | Uint32Array + | Int8Array + | Int16Array + | Int32Array + | BigUint64Array + | BigInt64Array + | Float32Array + | Float64Array + +type ZlibInputType = string | TypedArray | ArrayBuffer | DataView + +type HttpMethod = 'GET' | 'POST' | 'PUT' | 'HEAD' | 'DELETE' | 'PATCH' | string +type TrimRoute = T extends `${infer U}${'+' | '*'}` ? U : T +type ExtractPattern = T extends `${infer P}{${infer U}}${infer S}` ? TrimRoute | ExtractPattern

: never +export type CapturedPattern = string extends T ? Record : { [P in ExtractPattern]: string } +type SplitRoute = T extends `${infer M extends HttpMethod} ${infer P}` ? [M, P] : [string, T] + +// type X = ExtractPattern2<'/{foo}/{bar}'> +// type ExtractPattern2 = T extends `${infer P}{${infer U}}${infer S}` +// ? [...ExtractPattern2

, TrimRoute, ...ExtractPattern2] +// : [] + +export type PathArgs = T extends `${infer P}{${infer U}}${infer S}` + ? [...PathArgs

, string, ...PathArgs] + : [] + +export interface HttpRequest { + readonly path: string + readonly method: SplitRoute[0] + readonly headers: Request['headers'] + readonly cookies?: string[] + readonly context?: any + readonly queryString?: string + readonly pathParameters: CapturedPattern[1]> +} + + +// type FormDataValue = string | Blob | File +// type TypedFormData> = FormData & { +// [Symbol.iterator](): IterableIterator<[string, FormDataEntryValue]>; +// /** Returns an array of key, value pairs for every entry in the list. */ +// entries(): IterableIterator<[string, FormDataEntryValue]>; +// /** Returns a list of keys in the list. */ +// keys(): IterableIterator; +// /** Returns a list of values in the list. */ +// values(): IterableIterator; +// } + +// type TypedJsonBody = Omit & { json(): Promise } +// type TypedJsonResponse = Omit & { json(): Promise } + +type HandlerResponse = Response | R | void // JsonResponse | R | void +type HandlerArgs = U extends undefined ? [request: HttpRequest] : [request: HttpRequest, body: U] +export type HttpHandler = + (this: C, ...args: HandlerArgs) => Promise> | HandlerResponse + +export type HttpFetcher = + (...args: [...PathArgs, ...(U extends undefined ? [] : [body: U])]) => Promise + +// Only used to short-circuit APIG responses +/** @internal */ +export const kHttpResponseBody = Symbol.for('kHttpResponseBody') +// export class JsonResponse extends Response { +// readonly [kHttpResponseBody]: Uint8Array + +// public constructor(body: T, init?: ResponseInit | undefined) { +// const headers = new Headers(init?.headers) +// if (!headers.has('content-type')) { +// headers.set('content-type', 'application/json') +// } + +// const encoded = new TextEncoder().encode(JSON.stringify(body)) +// super(encoded, { ...init, headers }) + +// this[kHttpResponseBody] = encoded +// } +// } + +// export interface JsonResponse { +// json(): Promise +// } + +interface BindingBase { + from: string // JSONPath + to: string // JSONPath +} + +interface PathBinding extends BindingBase { + type: 'path' +} + +interface QueryBinding extends BindingBase { + type: 'query' +} + +interface HeaderBinding extends BindingBase { + type: 'header' +} + +interface BodyBinding extends BindingBase { + type: 'body' +} + +type HttpBinding = PathBinding | QueryBinding | HeaderBinding | BodyBinding + +// This is always `https` for now +export interface HttpRoute { + readonly host: string + readonly port?: number + readonly method: string + readonly path: string + readonly query?: string + readonly body?: any + readonly bindings: { + readonly request: HttpBinding[] + readonly response: HttpBinding[] + } +} + +// FIXME: make this more robust +function isRoute(obj: unknown): obj is HttpRoute { + return typeof obj === 'object' && !!obj && typeof (obj as any)['bindings'] === 'object' && !!(obj as any)['bindings'] +} + +export interface HttpResponse { + body?: any + statusCode?: number + headers?: Record +} + +export type SubstituteRoute = T extends `${infer L}{${infer U}}${infer R}` + ? Root extends true + ? (`${SubstituteRoute}${string}${SubstituteRoute}` | `${SubstituteRoute}${string}${SubstituteRoute}?${string}`) + : `${SubstituteRoute}${string}${SubstituteRoute}` + : Root extends true ? (`${T}?${string}` | T) : T + +export type RequestArgs = T extends HttpHandler ? + U extends undefined + ? [] + : any extends U ? [body?: any] : [body: U] : never + + +export interface HttpErrorFields { + /** + * @internal + * @deprecated + */ + statusCode: number + status: number +} + +export class HttpError extends Error { + public readonly fields: HttpErrorFields + public constructor(message: string, fields: Partial = {}) { + super(message) + this.fields = { status: 500, statusCode: 500, ...fields } + // Backwards compat + this.fields.statusCode = fields.statusCode ?? fields.status ?? 500 + } +} + +/** @internal */ +export type Middleware = ( + req: HttpRequest, + body: U, + next: HttpHandler +) => HttpResponse | Promise + +interface TypedRegExpExecArray> extends RegExpExecArray { + readonly groups: T +} + +/** @internal */ +export interface TypedRegexp> extends RegExp { + exec(string: string): TypedRegExpExecArray | null +} + +/** @internal */ +export interface RouteRegexp extends RegExp { + exec(string: string): TypedRegExpExecArray> | null +} + +export function buildRouteRegexp(path: T, prefix?: string): RouteRegexp { + const pattern = /{([A-Za-z0-9]+)([\+\*])?}/g + const searchPatterns: string[] = [] + let match: RegExpExecArray | null + let lastIndex = 0 + + // TODO: handle duplicates + while (match = pattern.exec(path)) { + const qualifer = match[2] + const name = match[1] + + searchPatterns.push(path.slice(lastIndex, match.index).replace(/\//g, '\\/')) + if (!qualifer) { + searchPatterns.push(`(?<${name}>[^\\/\\s:]+\\/?)`) + } else { + searchPatterns.push(`(?<${name}>[^\\s:]${qualifer})`) + } + lastIndex = pattern.lastIndex + } + + if (lastIndex < path.length) { + searchPatterns.push(path.slice(lastIndex)) + } + + searchPatterns.push('$') + + return new RegExp(`^${prefix ?? ''}` + searchPatterns.join('')) as RouteRegexp +} + +export function* matchRoutes(path: string, entries: readonly [pattern: RegExp, value: T][]) { + for (const [pattern, value] of entries) { + const match = pattern.exec(path) + + if (match) { + yield { value, match } + } + } +} + +function parseSegment(segment: string) { + const pattern = /{([A-Za-z0-9]+)([\+\*])?}/g + const parts: ({ type: 'literal'; value: string } | { type: 'pattern'; value: { qualifier?: '*' | '+' } })[] = [] + + let match: RegExpExecArray | null + let lastIndex = 0 + + while (match = pattern.exec(segment)) { + const qualifier = match[2] as '+' | '*' | undefined + const name = match[1] + + if (match.index !== 0) { + parts.push({ + type: 'literal', + value: segment.slice(lastIndex, match.index), + }) + } + + parts.push({ + type: 'pattern', + value: { qualifier }, + }) + + lastIndex = pattern.lastIndex + } + + if (lastIndex < segment.length) { + parts.push({ + type: 'literal', + value: segment.slice(lastIndex), + }) + } + + return parts +} + +// TODO: `+` should take precdence over `*` +// e.g. `/{foo+}` > `/{foo*}` +// or make it so these routes are incompatible. That's probably smarter. +function compareSegment(a: string, b: string): number { + const pa = parseSegment(a) + const pb = parseSegment(b) + + if (pa.length > pb.length) { + return -compareSegment(b, a) + } + + for (let i = 0; i < pa.length; i++) { + const ai = pa[i] + const bi = pb[i] + + if (ai.type === 'pattern' && bi.type === 'literal') { + return -1 + } else if (ai.type === 'literal' && bi.type === 'pattern') { + return 1 + } else if (ai.type === 'literal' && bi.type === 'literal') { + const d = ai.value.length - bi.value.length + if (d !== 0) { + return d + } + } + } + + if (pa[pa.length - 1].type === 'literal') { + return 1 + } + + return 0 +} + +export function compareRoutes(a: string, b: string) { + const diff = a.split('/').length - b.split('/').length + if (diff === 0) { + return compareSegment(a, b) + } + + return diff +} + +const TypedArray = Object.getPrototypeOf(Uint8Array) +function isTypedArray(obj: any) { + return obj instanceof TypedArray +} + +function resolveBody(body: any) { + if (body === undefined) { + return + } + + if (typeof body === 'object') { + if (body === null) { + return { + contentType: 'application/json', + body: 'null' + } + } + + if (isTypedArray(body)) { + const contentEncoding = body[kContentEncoding] + + return { + body, + contentEncoding, + } + } + + if (body instanceof URLSearchParams) { + return { + contentType: 'application/x-www-form-urlencoded', + body: body.toString() + } + } + + return { + contentType: 'application/json', + body: typeof Buffer !== 'undefined' + ? Buffer.from(JSON.stringify(body), 'utf-8').toString('utf-8') + : JSON.stringify(body) + } + } + + return { body } +} + +function parseContentType(header: string) { + const directives = header.split(';') + const mimeType = directives[0].trim() + const remainder = directives.slice(1).map(d => { + const match = d.trim().match(/(?.*)=(?.*)/) as TypedRegExpExecArray<{ key: string; value: string }> | null + + return match?.groups + }) + + const encoding = remainder.find(d => d?.key === 'charset')?.value + const boundary = remainder.find(d => d?.key === 'boundary')?.value + + return { + mimeType, + encoding, + boundary, + } +} + +function isJsonMimeType(mimeType: string) { + const match = mimeType.match(/application\/(?:([^+\s]+)\+)?json/) + + return !!match +} + +function filterUndefined>(obj: T): { [P in keyof T]+?: NonNullable } { + return Object.fromEntries(Object.entries(obj).filter(([k, v]) => v !== undefined)) as any +} + +function toLowerCase>(obj: T): { [P in keyof T & string as Lowercase

]+?: T[P] } { + return Object.fromEntries(Object.entries(obj).map(([k, v]) => [k.toLowerCase(), v])) as any +} + +const kContentEncoding = Symbol.for('contentEncoding') + +/** @internal */ +export function encode(data: ZlibInputType, encoding: 'gzip' | 'br' = 'gzip') { + const encodeFn = encoding === 'gzip' ? zlib.gzip : zlib.brotliCompress + + return new Promise((resolve, reject) => encodeFn(data, (err, res) => { + if (err) { + reject(err) + } else { + Object.defineProperty(res, kContentEncoding, { + value: encoding, + configurable: true, + enumerable: false, + writable: false, + }) + resolve(res) + } + })) +} + +function getDecodeFn(encoding: 'gzip' | 'br' | 'deflate') { + switch (encoding) { + case 'gzip': + return zlib.gunzip + case 'br': + return zlib.brotliDecompress + case 'deflate': + return zlib.deflate + default: + throw new Error(`Unsupported encoding: ${encoding}`) + } +} + +/** @internal */ +export function decode(data: ZlibInputType, encoding: 'gzip' | 'br' | 'identity') { + if (encoding === 'identity') { + return Promise.resolve(Buffer.from(data as string) as Uint8Array) + } + + const decodeFn = getDecodeFn(encoding) + + return new Promise((resolve, reject) => decodeFn(data, (err, res) => { + if (err) { + reject(err) + } else { + resolve(res) + } + })) +} + +// TODO: need caching to avoid infinitely recursive queries when adding the logger to the DNS service +function createDnsResolver() { + // http.globalAgent.options.lookup +} + +export function createFetcher(init?: Pick) { + async function doFetch(route: HttpRoute, ...args: T): Promise { + const { request, body } = applyRoute(route, args) + + if (init?.headers) { + request.headers ??= {} + request.headers = { ...init.headers, ...request.headers } + } + + return doRequest(request, body) + } + + return { fetch: doFetch } +} + +interface FetchInit { + body?: any + baseUrl?: string + method?: string + headers?: Record +} + +export async function fetch(route: HttpRoute, ...request: T): Promise +export async function fetch(url: string, init?: FetchInit): Promise +export async function fetch(urlOrRoute: string | HttpRoute, ...args: [init?: FetchInit] | any[]): Promise { + if (isRoute(urlOrRoute)) { + const { request, body } = applyRoute(urlOrRoute, args) + const resp = await doRequest(request, body) + // TODO: response bindings + + return resp + } + + let url = urlOrRoute + const init = args[0] as FetchInit + const method = init?.method ?? (!url.startsWith('/') && url.includes(' ') ? url.split(' ')[0] : 'GET') + if (!method) { + throw new Error(`No method provided`) + } + + if (url.includes(' ')) url = url.split(' ')[1] + if (init?.baseUrl) url = init.baseUrl + url + + const parsedUrl = new URL(url) + const resolvedBody = resolveBody(init?.body) + const headers = filterUndefined({ + 'content-type': resolvedBody?.contentType, + 'content-encoding': resolvedBody?.contentEncoding, + 'accept-encoding': 'br, gzip, deflate', + ...(init?.headers ? toLowerCase(init.headers) : undefined), + }) + + return doRequest({ + method, + headers, + host: parsedUrl.host, + hostname: parsedUrl.hostname, // XXX: added to make `localhost:${port}` work + port: parsedUrl.port, + path: parsedUrl.pathname, + protocol: parsedUrl.protocol, + search: parsedUrl.search, // not sure if this is correct + }, resolvedBody?.body) +} + +interface AppliedRoute { + method?: string + host?: string + port?: number + path?: string + protocol: string + headers?: Record +} + +export function applyRoute(route: HttpRoute, args: any[]) { + const pathBindings = route.bindings.request.filter(b => b.type === 'path') as PathBinding[] + const body = route.body !== undefined + ? substitutePaths(route.body, args) + : undefined + + const resolvedBody = resolveBody(body) + + const headerBindings = route.bindings.request.filter(b => b.type === 'header') as HeaderBinding[] + + const request: AppliedRoute = { + method: route.method, + host: route.host, + port: route.port, + path: subsitutePathTemplate(route.path, pathBindings, args), + // hostname: parsedUrl.hostname, // TODO: add this so it works w/ the VSC ext proxy agent + // XXX: hack to make local impl. work + protocol: route.host === 'localhost' ? 'http:' : 'https:', + headers: filterUndefined({ + ...resolveHeaderBindings(headerBindings), + 'content-type': resolvedBody?.contentType, + 'content-encoding': resolvedBody?.contentEncoding, + }) + } + + return { request, body: resolvedBody?.body } +} + +function resolveHeaderBindings(bindings: HeaderBinding[]) { + if (bindings.length === 0) { + return + } + + const headers: Record = {} + for (const b of bindings) { + if (b.from === 'randomUUID()') { + headers[b.to] = randomUUID() + } else { + throw new Error(`Unknown binding: ${b.from}`) + } + } + + return headers +} + +function randomUUID(): string { + return crypto.randomUUID() +} + +function randomUUIDBrowser() { + return self.crypto.randomUUID() +} + +addBrowserImplementation(randomUUID, randomUUIDBrowser) + + +function getResourceFromOptions(opt: http.RequestOptions) { + const protocol = opt.protocol ?? 'https:' + const host = `${opt.host}${opt.port ? `:${opt.port}` : ''}` ?? opt.hostname + if (host) { + return `${protocol}//${host}${opt.path ?? ''}` + } + + if (!opt.path) { + throw new Error('Unable to create URL from request: no pathname specified') + } + + return opt.path +} + +const _fetch = globalThis.fetch +async function sendFetchRequest(request: http.RequestOptions | URL, body?: any) { + if (request instanceof URL) { + return _fetch(request, { body }) + } + + const resource = getResourceFromOptions(request) + return _fetch(resource, { + body, + method: request.method, + headers: request.headers + ? new Headers(filterUndefined(request.headers) as Record) + : undefined, + }) +} + +async function doRequestBrowser(request: http.RequestOptions | URL, body?: any) { + const resp = await sendFetchRequest(request, body) + const contentTypeRaw = resp.headers.get('content-type') + const contentType = contentTypeRaw ? parseContentType(contentTypeRaw) : undefined + const isJson = contentType ? isJsonMimeType(contentType.mimeType) : false + + // TODO: handle content encodings other than utf-8 for JSON? + const data = await (isJson ? resp.json() : resp.text()) + + if (resp.status >= 400) { + if (isJson && data) { + throw Object.assign( + new Error(data.message ?? `Received non-2xx status code: ${resp.status} [${resp.statusText}]`), + { statusCode: resp.status }, + data + ) + } else { + throw Object.assign( + new Error(`Received non-2xx status code: ${resp.status} [${resp.statusText}]`), + { statusCode: resp.status, message: data }, + ) + } + } else if (resp.status === 302 || resp.status === 303) { + // TODO: follow redirect? + throw new Error(`Received redirect: ${resp.headers.get('location')}`) + } + + return data +} + +addBrowserImplementation(doRequest, doRequestBrowser) + +function doRequest(request: http.RequestOptions | URL, body?: any) { + // We create an error here to preserve the trace + const err = new Error() + + return new Promise((resolve, reject) => { + const http: typeof import('node:https') = request.protocol === 'http:' ? require('node:http') : require('node:https') + + const req = http.request(request, res => { + const contentType = res.headers['content-type'] ? parseContentType(res.headers['content-type']) : undefined + const contentEncoding = res.headers['content-encoding'] + const isJson = contentType ? isJsonMimeType(contentType.mimeType) : false + const buffers: Buffer[] = [] + + function close(val: any, code?: number) { + if (val instanceof Error) { + reject(val) + } else { + if (code === 204 && val === undefined) { + resolve(undefined) + } + + const r = val || {} + if (typeof r === 'object') { + Object.defineProperty(r, '$headers', { + value: res.headers, + enumerable: false + }) + } + resolve(r) + } + + res.destroy() + } + + res.on('data', d => buffers.push(d)) + res.on('error', close) + res.on('end', async () => { + const result = Buffer.concat(buffers) + const decoded = contentEncoding + ? ((await decode(result, contentEncoding as any) as Buffer)) // XXX: `as Buffer` is not safe + : result + + if (res.statusCode && res.statusCode >= 400) { + if (isJson && result) { + const e = JSON.parse(decoded.toString('utf-8')) + + err.message = e.message ?? `Received non-2xx status code: ${res.statusCode}` + const stack = `${e.name ?? 'Error'}: ${err.message}\n` + err.stack?.split('\n').slice(1).join('\n') + close(Object.assign(err, { statusCode: res.statusCode, stack }, e)) + } else { + err.message = decoded.toString('utf-8') + const stack = `${err.name ?? 'Error'}: ${err.message}\n` + err.stack?.split('\n').slice(1).join('\n') + close(Object.assign(err, { statusCode: res.statusCode, stack })) + } + } else if (res.statusCode === 302 || res.statusCode === 303 || res.statusCode === 304) { + close({ + statusCode: res.statusCode, + headers: res.headers, + }) + } else { + if (isJson) { + close(decoded ? JSON.parse(decoded.toString('utf-8')) : undefined, res.statusCode) + } else if (res.headers['content-type'] === 'application/octet-stream') { + close(decoded, res.statusCode) + } else { + close(decoded.toString('utf-8'), res.statusCode) + } + } + }) + }) + req.on('error', reject) + req.end(body) + }) +} + +function substitutePaths(template: any, args: any[]): any { + if (typeof template === 'string') { + return getPath(args, template) + } + + if (Array.isArray(template)) { + return template.map(x => substitutePaths(x, args)) + } + + if (typeof template === 'object' && !!template) { + const res = { ...template } + for (const [k, v] of Object.entries(res)) { + res[k] = substitutePaths(v, args) + } + + return res + } + + return template +} + +function createBody(bodyTemplate: any, bindings: BodyBinding[], args: any[]) { + +} + +export function createPathBindings(pathTemplate: string) { + const pattern = /{([A-Za-z0-9]+\+?)}/g + const result: PathBinding[] = [] + let match: RegExpExecArray | null + + while (match = pattern.exec(pathTemplate)) { + result.push({ + type: 'path', + from: `$[${result.length}]`, // This is the naive solution (no duplicates) + to: match[1], + }) + } + + return result +} + +function subsitutePathTemplate(pathTemplate: string, bindings: PathBinding[], args: any[]) { + const pattern = /{([A-Za-z0-9]+\+?)}/g + const result: string[] = [] + let match: RegExpExecArray | null + let lastIndex = 0 + + while (match = pattern.exec(pathTemplate)) { + const binding = bindings.find(b => b.to === match![1]) + if (!binding) { + throw new Error(`Missing binding for path pattern: ${match[1]}`) + } + + result.push(pathTemplate.slice(lastIndex, match.index)) + + const val = getPath(args, binding.from) + if (typeof val !== 'string') { + throw new Error(`Invalid path binding for "${match[1]}". Expected type string, got ${typeof val}`) + } + + result.push(val) + lastIndex = pattern.lastIndex + } + + if (lastIndex < pathTemplate.length) { + result.push(pathTemplate.slice(lastIndex)) + } + + return result.join('') +} + +// JSON path expressions + +function assignPath(state: Record | any[], p: string, val: any) { + let didInit = false + let current: typeof state | undefined + let lastKey: string | undefined + const scanner = createJsonPathScanner(p) + for (const key of scanner.scan()) { + if (!didInit) { + if (key !== '$') { + throw new Error(`Expected expression to start with '$'`) + } + + current = state + didInit = true + } else { + if (lastKey !== undefined) { + current = (current as any)[lastKey] ??= (isNaN(Number(key)) ? {} : []) + } + lastKey = key + } + } + + if (!lastKey) { + throw new Error(`Cannot set value at root node`) + } + + (current as any)[lastKey] = val + + return state +} + +function getPath(state: Record | any[], p: string) { + let didInit = false + let current: typeof state | undefined + const scanner = createJsonPathScanner(p) + for (const key of scanner.scan()) { + if (!didInit) { + if (key !== '$') { + throw new Error(`Expected expression to start with '$'`) + } + + current = state + didInit = true + } else { + current = (current as any)?.[key] + } + } + + return current +} + +function createJsonPathScanner(expression: string) { + let pos = 0 + + // This state is entered when encountering a '[' + function parseElementKey() { + const c = expression[pos] + if (c === "'" || c === '"') { + for (let i = pos + 1; i < expression.length; i++) { + if (expression[i] === c) { + if (expression[i + 1] !== ']') { + throw new Error(`Expected closing bracket at position ${pos}`) + } + + const token = expression.slice(pos + 1, i) + pos = i + 2 + + return token + } + } + } else { + if (c < '0' || c > '9') { + throw new Error(`Expected a number at position ${pos}`) + } + + for (let i = pos + 1; i < expression.length; i++) { + const c = expression[i] + if (c === ']') { + const token = expression.slice(pos, i) + pos = i + 1 + + return token + } + + if (c < '0' || c > '9') { + throw new Error(`Expected a number at position ${pos}`) + } + } + } + + throw new Error(`Malformed element access at position ${pos}`) + } + + function parseIdentifer() { + for (let i = pos; i < expression.length; i++) { + const c = expression[i] + if (c === '[' || c === '.') { + const token = expression.slice(pos, i) + pos = i + + return token + } + } + + const token = expression.slice(pos) + pos = expression.length + + return token + } + + function* scan() { + if (pos === expression.length) { + return + } + + while (pos < expression.length) { + const c = expression[pos] + if (c === '[') { + pos += 1 + + yield parseElementKey() + } else if (c === '.') { + pos += 1 + + yield parseIdentifer() + } else { + yield parseIdentifer() + } + } + } + + return { scan } +} + + +export function getContentType(p: string) { + return getMimeType(p) +} + +const mimeTypes = { + '.html': 'text/html', + '.css': 'text/css', + '.js': 'application/javascript', + '.mjs': 'application/javascript', + '.cjs': 'application/javascript', + '.json': 'application/json', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.gif': 'image/gif', + '.wav': 'audio/wav', + '.mp4': 'video/mp4', + '.pdf': 'application/pdf', + '.doc': 'application/msword', + '.xls': 'application/vnd.ms-excel', + '.ppt': 'application/vnd.ms-powerpoint', + '.ico': 'image/x-icon', + '.svg': 'image/svg+xml', + '.csv': 'text/csv', + '.txt': 'text/plain', + '.zip': 'application/zip', + '.xml': 'application/xml', + '.mp3': 'audio/mpeg', + '.mpeg': 'video/mpeg', + '.tiff': 'image/tiff', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.odt': 'application/vnd.oasis.opendocument.text', + '.ods': 'application/vnd.oasis.opendocument.spreadsheet', + '.odp': 'application/vnd.oasis.opendocument.presentation', + '.rar': 'application/x-rar-compressed', + '.tar': 'application/x-tar', + '.rtf': 'application/rtf', + '.bmp': 'image/bmp', + '.psd': 'image/vnd.adobe.photoshop', + '.avi': 'video/x-msvideo', + '.mov': 'video/quicktime', + '.flv': 'video/x-flv', + '.ttf': 'font/ttf', + '.otf': 'font/otf', + '.webp': 'image/webp', + '.woff': 'font/woff', + '.woff2': 'font/woff2', +} as Record + +function getMimeType(p: string) { + const dotIndex = p.lastIndexOf(".") + const extension = dotIndex === -1 ? p : p.slice(dotIndex) + + return mimeTypes[extension] || 'application/octet-stream' +} \ No newline at end of file diff --git a/src/runtime/modules/key-service.ts b/src/runtime/modules/key-service.ts new file mode 100644 index 0000000..197941b --- /dev/null +++ b/src/runtime/modules/key-service.ts @@ -0,0 +1,330 @@ +//@internal +//# moduleId = synapse:key-service +//# transform = persist + +import { defineResource } from 'synapse:core' +import * as crypto from 'node:crypto' +import * as storage from 'synapse:srl/storage' +import { HttpError } from 'synapse:http' + +export function createKeyService() { + const keyBucket = new storage.Bucket() + + class EdwardsKeyPair extends defineResource({ + create: async () => { + const id = crypto.randomUUID() + const { publicKey, privateKey } = await createEdwardsKeyPair() + + await Promise.all([ + keyBucket.put(`${id}.public`, Buffer.from(publicKey)), + keyBucket.put(`${id}.private`, Buffer.from(privateKey)), + ]) + + return { id, alg: 'EdDSA', crv: 'Ed448' } + }, + update: state => state, + delete: async state => { + await Promise.all([ + keyBucket.delete(`${state.id}.public`), + keyBucket.delete(`${state.id}.private`) + ]) + } + }) { + async sign(data: string | Buffer) { + const location = `${this.id}.private` + const keyData = Buffer.from(await keyBucket.get(location)) + const privateKey = await importPrivateEdwardsKey(keyData) + const buffer = typeof data === 'string' ? Buffer.from(data) : data + + return crypto.subtle.sign('Ed448', privateKey, buffer) + } + + async verify(data: string | Buffer, signature: Buffer) { + const keyData = await this.getPublicKey() + const publicKey = await importPublicEdwardsKey(keyData) + const buffer = typeof data === 'string' ? Buffer.from(data) : data + + return crypto.subtle.verify('Ed448', publicKey, signature, buffer) + } + + async getPublicKey() { + const location = `${this.id}.public` + const keyData = Buffer.from(await keyBucket.get(location)) + + return keyData + } + + async getPublicWebKey() { + const keyData = await this.getPublicKey() + const publicKey = await crypto.subtle.importKey('raw', keyData, { name: 'Ed448' }, true, ['verify']) + const jwk = await crypto.subtle.exportKey('jwk', publicKey) + + return jwk + } + } + + class RSAKeyPair extends defineResource({ + create: async () => { + const id = crypto.randomUUID() + const result = crypto.generateKeyPairSync('rsa', { + modulusLength: 4096 + }) + + const publicKey = result.publicKey.export({ type: 'pkcs1', format: 'pem' }) + const privateKey = result.privateKey.export({ type: 'pkcs8', format: 'pem' }) + + await Promise.all([ + keyBucket.put(`${id}.public`, publicKey), + keyBucket.put(`${id}.private`, privateKey) + ]) + + return { id, alg: 'RS256', crv: undefined } + }, + update: state => state, + delete: async state => { + await Promise.all([ + keyBucket.delete(`${state.id}.public`), + keyBucket.delete(`${state.id}.private`) + ]) + } + }) { + async sign(data: string | Buffer) { + const location = `${this.id}.private` + const privateKey = crypto.createPrivateKey(Buffer.from(await keyBucket.get(location))) + const signer = crypto.createSign('RSA-SHA256') + + return signer.update(data).sign(privateKey) + } + + async verify(data: string | Buffer, signature: Buffer) { + const location = `${this.id}.public` + const publicKey = crypto.createPublicKey(Buffer.from(await keyBucket.get(location))) + const verifier = crypto.createVerify('RSA-SHA256') + + return verifier.update(data).verify(publicKey, signature) + } + + async getPublicKey() { + const location = `${this.id}.public` + const keyData = Buffer.from(await keyBucket.get(location)) + + return keyData + } + + async getPublicWebKey() { + const keyData = await this.getPublicKey() + const publicKey = crypto.createPublicKey(keyData) + + return publicKey.export({ format: 'jwk' }) + } + } + + return { RSAKeyPair, EdwardsKeyPair } +} + +type ServiceResources = ReturnType +export type RSAKeyPair = InstanceType +export type EdwardsKeyPair = InstanceType + +// From `lib.dom.d.ts` +export interface RsaOtherPrimesInfo { + d?: string; + r?: string; + t?: string; +} + +// From `lib.dom.d.ts` +export interface JsonWebKey { + alg?: string; + crv?: string; + d?: string; + dp?: string; + dq?: string; + e?: string; + ext?: boolean; + k?: string; + key_ops?: string[]; + kty?: string; + n?: string; + oth?: RsaOtherPrimesInfo[]; + p?: string; + q?: string; + qi?: string; + use?: string; + x?: string; + y?: string; +} + +export interface KeyPair { + readonly alg: string + readonly crv?: string + sign(data: string | Buffer): Promise + verify(data: string | Buffer, signature: Buffer): Promise + getPublicWebKey(): Promise +} + +export async function createEdwardsKeyPair() { + const result = (await crypto.subtle.generateKey({ + name: 'Ed448', + }, true, ['sign', 'verify'])) as crypto.webcrypto.CryptoKeyPair + + const publicKey = await crypto.subtle.exportKey('raw', result.publicKey) + const privateKey = await crypto.subtle.exportKey('pkcs8', result.privateKey) + + return { publicKey, privateKey } +} + +export async function importPublicEdwardsKey(keyData: Buffer) { + const key = await crypto.subtle.importKey('raw', keyData, { name: 'Ed448' }, true, ['verify']) + + return key +} + +export async function importPrivateEdwardsKey(keyData: Buffer) { + const key = await crypto.subtle.importKey('pkcs8', keyData, { name: 'Ed448' }, true, ['sign']) + + return key +} + +export async function createJwsBody(url: string, payload: any, key: KeyPair, nonce: string, kid?: string) { + const baseHeader = { + alg: key.alg, + crv: key.crv, + } + const header = kid + ? { ...baseHeader, kid, nonce, url } + : { ...baseHeader, jwk: await key.getPublicWebKey(), nonce, url } + + const encodedHeader = Buffer.from(JSON.stringify(header)).toString('base64url') + const encodedPayload = payload ? Buffer.from(JSON.stringify(payload)).toString('base64url') : '' + const encoded = encodedHeader + '.' + encodedPayload + const signature = Buffer.from(await key.sign(encoded)).toString('base64url') + + return { + protected: encodedHeader, + payload: encodedPayload, + signature, + } +} + +export async function thumbprintBase64Url(jwk: JsonWebKey) { + const hash = await crypto.subtle.digest('SHA-256', Buffer.from(JSON.stringify(getData()))) + + return Buffer.from(hash).toString('base64url') + + function getData() { + switch (jwk.kty) { + case 'RSA': + return { + e: jwk.e, + kty: jwk.kty, + n: jwk.n, + } + case 'EC': + return { + crv: jwk.crv, + kty: jwk.kty, + x: jwk.x, + y: jwk.y + } + case 'oct': + return { + k: jwk.k, + kty: jwk.kty, + } + default: + throw new Error(`Unknown key type: ${jwk.kty}`) + } + } +} + +// TODO: dedupe with `createJwsBody` +export async function encodeJwt(payload: any, key: Pick) { + const header = { alg: key.alg, crv: key.crv } + const encodedHeader = Buffer.from(JSON.stringify(header)).toString('base64url') + const encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64url') + const encoded = encodedHeader + '.' + encodedPayload + const signature = Buffer.from(await key.sign(encoded)).toString('base64url') + + return `${encoded}.${signature}` +} + +interface JwtClaims { + iss?: string + sub?: string + aud?: string | string[] + exp?: number + nbf?: number + iat?: number + jti?: string +} + +// JWTs kind of suck. There's no need to encode so much information into +// the token when using centralized auth. +export async function createJwt( + key: Pick, + durationInMinutes: number, + claims: Omit +) { + const payload = { + iat: Math.floor(Date.now() / 1000) - 60, + exp: Math.floor(Date.now() / 1000) + (durationInMinutes * 60), + // jti: claims.jti ?? crypto.randomUUID(), + ...claims, + } + + return { + token: await encodeJwt(payload, key), + expirationEpoch: Date.now() + ((durationInMinutes - 1) * 60 * 1000), // 1 minute offset for clock drift + } +} + +export async function getClaims(token: string, secret: Pick) { + const [header, body, signature] = token.split('.') + const decodedHeader = JSON.parse(Buffer.from(header, 'base64url').toString('utf-8')) + if (decodedHeader.alg !== secret.alg) { + throw new Error(`Invalid JWT header. Expected "alg" to be "${secret.alg}".`) + } + + if (secret.crv && decodedHeader.crv !== secret.crv) { + throw new Error(`Invalid JWT header. Expected "crv" to be "${secret.crv}".`) + } + + const isValidSignature = await secret.verify(header + '.' + body, Buffer.from(signature, 'base64url')) + if (!isValidSignature) { + throw new HttpError('Bad JWT signature', { statusCode: 401 }) + } + + return JSON.parse(Buffer.from(body, 'base64url').toString('utf-8')) as JwtClaims +} + +export function isJwtExpired(claims: JwtClaims) { + if (!claims.exp) { + return false + } + + const currentTimeSeconds = Date.now() / 1000 + + return claims.exp <= currentTimeSeconds +} + +export class RandomString extends defineResource({ + create: (length: number) => { + const value = crypto.randomBytes(length).toString('base64url') + + return { value } + }, + update: state => state, +}) {} + + +export async function buildDnsTxtValue(token: string, key: EdwardsKeyPair | RSAKeyPair) { + const thumbprint = await thumbprintBase64Url(await key.getPublicWebKey()) + const hash = await crypto.subtle.digest('SHA-256', Buffer.from(token + '.' + thumbprint)) + + return Buffer.from(hash).toString('base64url') +} + + +// MacOS profiles +// https://developer.apple.com/business/documentation/Configuration-Profile-Reference.pdf diff --git a/src/runtime/modules/lib.ts b/src/runtime/modules/lib.ts new file mode 100644 index 0000000..3ca91c9 --- /dev/null +++ b/src/runtime/modules/lib.ts @@ -0,0 +1,336 @@ +//# moduleId = synapse:lib +//# transform = persist + +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import * as core from 'synapse:core' +import { Fn, peekNameSym } from 'synapse:terraform' +import { readFileSync } from 'node:fs' + +interface BundleOptions { + readonly external?: string[] + readonly destination?: string + readonly moduleTarget?: 'cjs' | 'esm' | 'iife' + readonly platform?: 'node' | 'browser' + readonly minify?: boolean + readonly immediatelyInvoke?: boolean + readonly banners?: string[] + + // For artifacts + /** @internal */ + readonly publishName?: string + /** @internal */ + readonly includeAssets?: boolean + + /** + * @internal + * Generates code to lazily load modules + */ + // readonly lazyLoad?: string[] +} + +export enum AssetType { + File = 0, + Directory = 1, + Archive = 2, +} + +// FIXME: don't overload `Bundle` with closures?? +// TODO: if `target` is a string then we are just bundling a file vs. a JavaScript object/function (???) +//# resource = true +export class Bundle extends core.Closure { + public constructor(target: ((...args: any[]) => any) | Record, opt?: BundleOptions) { + const normalizedLocation = opt?.destination + ? path.relative(core.cwd(), opt.destination) + : undefined + + super({ + location: normalizedLocation, + options: { ...opt, destination: normalizedLocation }, + captured: new core.SerializedObject(target).filePath, + }) + + Object.assign(this, { destination: tagPointer(this.destination) }) + } +} + +//# resource = true +/** @internal */ +export class Export extends core.Closure { + public constructor(target: any, opt?: { id?: string, destination?: string, source?: string, testSuiteId?: number; publishName?: string }) { + const normalizedLocation = opt?.destination + ? path.relative(core.cwd(), opt.destination) + : undefined + + super({ + source: opt?.source, + location: normalizedLocation, + options: { + ...opt, + bundled: false, + destination: normalizedLocation, + }, + captured: new core.SerializedObject(target).filePath, + }) + + Object.assign(this, { destination: tagPointer(this.destination) }) + } +} + +interface AssetProps { + readonly path: string + readonly type?: AssetType + readonly extname?: string + readonly extraFiles?: Record + // readonly destination?: string +} + +class AssetConstruct extends core.Asset { + public constructor(props: AssetProps) { + super({ + path: props.path, + extname: props.extname, + type: props.type ?? AssetType.Archive, + // extraFiles: props.extraFiles, + // filePath: props.destination, + }) + + // Would be better if this was 'lazy_refresh' or something similar + core.updateLifecycle(this, { force_refresh: true }) + } +} + +//# resource = true +export class Archive extends AssetConstruct { + public constructor(pathOrTarget: string | Bundle) { + const filePath = pathOrTarget instanceof Bundle ? pathOrTarget.destination : pathOrTarget as string + const extname = pathOrTarget instanceof Bundle ? pathOrTarget.extname : path.extname(pathOrTarget as string) + //const extraFiles = pathOrTarget instanceof Bundle ? (pathOrTarget as any).assets : undefined + + super({ + extname, + path: filePath, + type: AssetType.Archive, + // extraFiles, + }) + } +} + +export async function readAsset(asset: string) { + if (asset.startsWith('file:')) { + // XXX: ideally we'd resolve in the bundle + const data = await fs.readFile(path.resolve(__dirname, asset.slice('file:'.length))) + + return Buffer.from(data).toString('utf-8') + } + + const artifactFs = core.getArtifactFs() + const data = await artifactFs.readArtifact(asset) + + return Buffer.from(data).toString('utf-8') +} + +const pointerSymbol = Symbol.for('synapse.pointer') +function tagPointer(ref: string): string { + return Object.assign(ref as any, { [pointerSymbol]: true }) +} + +const nodeEnv = process.env['NODE_ENV'] +export function isProd() { + return nodeEnv === 'production' || envName?.includes('production') +} + +const envName = process.env['SYNAPSE_ENV'] +export function getEnvironmentName() { + return envName +} + +/** @internal */ +export interface FileAsset { + readonly pointer: core.DataPointer + read(): Promise + resolve(): Promise +} + +//# resource = true +/** @internal */ +export function createFileAsset(filePath: string, opt?: { publishName?: string }): FileAsset { + const data = readFileSync(filePath) + const artifactFs = core.getArtifactFs() + const pointer = opt?.publishName + ? artifactFs.writeFileSync(opt.publishName, data) + : artifactFs.writeArtifactSync(data, opt) + + async function read() { + const artifactFs = core.getArtifactFs() + const data = await artifactFs.readArtifact(pointer) + + return Buffer.from(data).toString('utf-8') + } + + async function resolve() { + const artifactFs = core.getArtifactFs() + + return artifactFs.resolveArtifact(pointer) //, { extname }) + } + + return { + pointer, + read, + resolve, + } +} + +class DataAsset extends core.defineResource({ + create: async (data: string, extname?: string) => { + const artifactFs = core.getArtifactFs() + const pointer = await artifactFs.writeArtifact(Buffer.from(data, 'utf-8')) + + return { pointer, extname } + }, +}) { + async read() { + const artifactFs = core.getArtifactFs() + const data = await artifactFs.readArtifact(this.pointer) + + return Buffer.from(data).toString('utf-8') + } +} + +/** @internal */ +export function createDataAsset(data: string, extname?: string) { + const asset = new DataAsset(data, extname) + + return { + pointer: tagPointer(asset.pointer), + extname: asset.extname, + } +} + +/** @internal */ +export const resolveArtifact = core.defineDataSource(async (pointer: core.DataPointer, extname?: string) => { + const { hash } = pointer.resolve() + const resolved = await core.getArtifactFs().resolveArtifact(pointer, { extname }) + + return { hash, filePath: resolved } +}) + +// Target _must_ be a resource +/** @internal */ +export function addReplacementHook(target: T, hook: core.ReplacementHook) { + const handler = new Export(hook) + + core.updateLifecycle(target, { + hook: [{ + kind: 'replace', + input: target, + handler: handler.destination, + }] + }) +} + +const _calculateFromState = core.defineDataSource(async (resourceId: string, fn: (state: any) => any) => { + const client = core.getBackendClient() + const state = await client.getState(resourceId) + + return fn(state) +}) + +//# resource = true +/** @internal */ +export function calculateFromState, U>( + target: new (...args: any[]) => T, + fn: (state: T) => Promise | U +): any { + return _calculateFromState(core.peekResourceId(target), fn) +} + +function toSnakeCase(str: string) { + return str.replace(/[A-Z]/g, s => `_${s.toLowerCase()}`) +} + +interface GeneratedIdentifierProps { + readonly sep?: string + readonly prefix?: string + readonly maxLength?: number +} + +/** @internal */ +export class GeneratedIdentifier extends core.defineResource({ + create: async (props: GeneratedIdentifierProps) => { + return { + props, + value: generateName(props.maxLength, props.sep, props.prefix), + } + }, + update: async (state, props) => { + if ( + state.props.sep === props.sep && + state.props.prefix === props.prefix && + state.props.maxLength === props.maxLength + ) { + return state + } + + return { + props, + value: generateName(props.maxLength, props.sep, props.prefix), + } + }, +}) {} + +/** @internal */ +export function createGeneratedIdentifier(props: GeneratedIdentifierProps = {}) { + const ident = new GeneratedIdentifier(props) + + return ident.value +} + +let nameCounter = 0 +let previousTime: number +function generateName(maxLength?: number, sep = '-', namePrefix = 'synapse') { + const currentTime = Date.now() + if (currentTime !== previousTime) { + nameCounter = 0 + previousTime = currentTime + } + + const id = `${currentTime}${sep}${++nameCounter}` + const offset = namePrefix ? namePrefix.length + sep.length : 0 + const trimmed = maxLength ? id.slice(Math.max(0, id.length - (maxLength - offset))) : id + + return namePrefix ? `${namePrefix}${sep}${trimmed}` : trimmed +} + +//# resource = true +/** @internal */ +export function generateIdentifier, K extends keyof T>( + target: new (...args: any[]) => T, + attribute: K & string, + maxLength = 100, + sep?: string +): string { + if (!(peekNameSym in target)) { + throw new Error(`Unable to get resource id from target`) + } + + const resourceId = (target[peekNameSym] as any)() + + return Fn.generateidentifier(resourceId, toSnakeCase(attribute), maxLength, sep) +} + +interface BuildTarget { + readonly programId: string + readonly deploymentId: string +} + +declare var __buildTarget: BuildTarget + +/** @internal */ +export const getBuildTarget = core.defineDataSource(() => { + if (typeof __buildTarget === 'undefined') { + throw new Error(`Not within a build context`) + } + + return __buildTarget +}) \ No newline at end of file diff --git a/src/runtime/modules/reify.ts b/src/runtime/modules/reify.ts new file mode 100644 index 0000000..473e7f0 --- /dev/null +++ b/src/runtime/modules/reify.ts @@ -0,0 +1,18 @@ +//@internal +//# moduleId = synapse:reify + +import type { Schema, TypedObjectSchema, TypedArraySchema, TypedStringSchema, TypedNumberSchema } from 'synapse:validation' + + +export declare function schema(): never +export declare function schema(): TypedArraySchema +export declare function schema(): TypedObjectSchema +export declare function schema(): TypedStringSchema +export declare function schema(): TypedNumberSchema + +/** @internal */ +export function __schema(obj: any): any { + return obj +} + +// export declare function check(val: unknown): asserts val is T diff --git a/src/runtime/modules/serdes.ts b/src/runtime/modules/serdes.ts new file mode 100644 index 0000000..fc48da2 --- /dev/null +++ b/src/runtime/modules/serdes.ts @@ -0,0 +1,561 @@ +// Notes: +// The super.property syntax can be used inside a static block to reference static properties of a super class. +// class A { static foo = 'bar' }; class B extends A { static { getTerminalLogger().log(super.foo); } } + +// Private properties with the same name within different classes are entirely different and do not +// interoperate with each other. See them as external metadata attached to each instance, managed by the class. + +// Private fields/methods are basically non-serializable without modifying the source code. +// This is because the constructor _must_ be called for initialization to happen + +export const moveable = Symbol.for('__moveable__') +const moveableStr = `@@${moveable.description as '__moveable__'}` as const + +export const fqn = Symbol.for('__fqn__') +const pointerSymbol = Symbol.for('synapse.pointer') + +// async not supported yet +export const serializeSymbol = Symbol.for('__serialize__') + +export interface ExternalValue { + readonly id?: number | string + readonly hash?: string + readonly location?: string + readonly module: string // Not present with `reflection` + readonly valueType?: 'function' | 'reflection' | 'object' | 'resource' | 'regexp' | 'binding' | 'data-pointer' | 'bound-function' + readonly captured?: any[] // If present, the module export will be invoked using these as args + readonly export?: string + readonly parameters?: any[] // Used to invoke a ctor or function + readonly operations?: ReflectionOperation[] // Only applicable to `reflection` + readonly symbols?: Record + readonly packageInfo?: PackageInfo // Only used for `reflection` type right now + readonly decorators?: any[] + + // Refers to the values of a bound function + readonly boundTarget?: any + readonly boundThisArg?: any + readonly boundArgs?: any[] + + // For reference bindings + readonly key?: string + readonly value?: number | string +} + +export interface PackageInfo { + readonly type: 'npm' | 'cspm' | 'synapse-provider' | 'file' | 'jsr' | 'synapse-tool' | 'spr' | 'github' + readonly name: string + readonly version: string + readonly packageHash?: string // Only relevant for `synapse` + readonly os?: string[] + readonly cpu?: string[] + readonly bin?: string | Record + readonly resolved?: { + readonly url: string + readonly integrity?: string + readonly isStubPackage?: boolean + readonly isSynapsePackage?: boolean + } +} + +export interface DependencyTree { + [name: string]: { + readonly location: string + readonly packageInfo: PackageInfo + readonly versionConstraint: string + readonly dependencies?: DependencyTree + readonly optional?: boolean + } +} + +interface RegExpValue extends ExternalValue { + readonly valueType: 'regexp' + readonly source: string + readonly flags: string +} + +interface DataPointerValue extends ExternalValue { + readonly valueType: 'data-pointer' + readonly hash: string + readonly storeHash: string +} + +export interface SerializedObject { + readonly id: string | number + readonly valueType: 'object' + readonly properties?: Record + readonly descriptors?: Record + readonly privateFields?: any[] + readonly prototype?: any // Object.prototype by default + readonly constructor?: any +} + +export interface ResourceValue { + readonly id: string | number + readonly valueType: 'resource' + readonly value: any +} + +interface GetOperation { + readonly type: 'get' + readonly property: string // can be a symbol? +} + +interface ConstructOperation { + readonly type: 'construct' + readonly args: any[] + // `newTarget` is only needed if we try to construct a derived class given the base ctor +} + +interface ApplyOperation { + readonly type: 'apply' + readonly args: any[] + readonly thisArg?: any +} + +interface ImportOperation { + readonly type: 'import' + readonly module: string + readonly location?: string + + readonly packageInfo?: PackageInfo + readonly dependencies?: DependencyTree +} + +interface GlobalOperation { + readonly type: 'global' +} + +export type ReflectionOperation = GetOperation | ConstructOperation | ApplyOperation | ImportOperation | GlobalOperation + +export type DataPointer = string & { + readonly ref: string + readonly hash: string; + readonly [pointerSymbol]: true + resolve(): { hash: string; storeHash: string } + isResolved(): boolean + isContainedBy(storeId: string): boolean +} + +const pointerPrefix = 'pointer:' + +function createPointer(hash: string, storeHash: string): DataPointer { + function resolve() { + return { hash, storeHash } + } + + const ref = `${pointerPrefix}${hash}` + + return Object.assign(ref, { + ref, + hash, + resolve, + isResolved: () => true, + isContainedBy: (storeId: string) => false, + [pointerSymbol]: true as const, + }) +} + + +interface ModuleLoader { + loadModule: (specifier: string, importer?: string) => any +} + +export function resolveValue( + value: any, + moduleLoader: ModuleLoader, + dataTable: Record = {}, + context = globalThis, + preserveId = false +): any { + function loadFunction(val: any, params: any[]) { + switch (params.length) { + case 1: return val(params[0]) + case 2: return val(params[0], params[1]) + case 3: return val(params[0], params[1], params[2]) + case 4: return val(params[0], params[1], params[2], params[3]) + default: return val(...params) + } + } + + function loadValueFromExports(desc: ExternalValue, exports: any): any { + const exported = desc.export + ? exports[desc.export] + : 'default' in exports ? exports.default : exports // XXX: this 'default' check is for backwards compat + + if (desc.captured) { + if (desc.captured instanceof Promise) { + return desc.captured.then(args => loadFunction(exported, args)) + } + return loadFunction(exported, desc.captured) + } + + return exported + } + + function reflectionWorker(operation: ReflectionOperation, operand?: any) { + if (operand === undefined && operation.type !== 'import' && operation.type !== 'global') { + throw new Error(`Bad operation, expected import or global to initialize operand: ${JSON.stringify(operation, undefined, 4)}`) + } + + switch (operation.type) { + case 'global': + return context + case 'import': + return moduleLoader.loadModule(operation.module, operation.location) + case 'get': + return operand[operation.property] + + // We need to serialize the receiver for this to work correctly + // return Reflect.get(operand, operation.property) + case 'apply': + return Reflect.apply(operand, operation.thisArg, operation.args) + case 'construct': + return Reflect.construct(operand, operation.args) + default: + throw new Error(`Unhandled operation: ${JSON.stringify(operation, undefined, 4)}`) + } + } + + function handleReflection(operations: ReflectionOperation[]) { + let operand: any + + for (const op of operations) { + if (operand instanceof Promise) { + operand = operand.then(x => reflectionWorker(op, x)) + } else { + operand = reflectionWorker(op, operand) + } + } + + return operand + } + + const deserialize = Symbol.for('deserialize') + function createObject(prototype: any, properties: any, descriptors: any, privateFields: any, constructor: any) { + const proto = prototype !== undefined + ? prototype + : constructor === undefined + ? Object.prototype + : constructor.prototype + + const obj = Object.create(proto ?? null, descriptors) + if (properties !== undefined) { + Object.assign(obj, properties) // Careful: this triggers `set` accessors, better to map as descriptors? + } + + if (constructor !== undefined) { + obj.constructor = constructor + } + + if (privateFields !== undefined && deserialize in obj) { + obj[deserialize]({ __privateFields: privateFields }) + } + + return obj + } + + function loadObject(desc: SerializedObject): any { + const proto = desc.prototype + const props = desc.properties + const privateFields = desc.privateFields + const descriptors = desc.descriptors + const constructor = Object.getOwnPropertyDescriptor(desc, 'constructor')?.value + + const all = [proto, props, descriptors, privateFields, constructor] as [any, any, any, any, any] + if (all.some(x => x instanceof Promise)) { + return Promise.all(all).then(args => createObject(...args)) + } + + return createObject(...all) + } + + function loadValue(desc: ExternalValue, exports: any): any { + if (desc.valueType === 'function' && desc.parameters) { + const params = desc.parameters + const f = loadValueFromExports(desc, exports) + if (f instanceof Promise) { + return f.then(fn => loadFunction(fn, params)) + } + + return loadFunction(f, params) + } + + return loadValueFromExports(desc, exports) + } + + const resolveCache = new Map() + const objectCache = new Map() // Used for re-hydrating refs of the form `{ id: number | string }` + const lateBindings = new Map() + + function resolve(o: any): any { + if (Array.isArray(o)) { + let isPromise = false + const a = o.map(x => { + const e = resolve(x) + isPromise ||= e instanceof Promise + + return e + }) + + return isPromise ? Promise.all(a) : a + } + + if (o === null || typeof o !== 'object') { + return o + } + + if (o[pointerSymbol]) { + return o + } + + if (objectCache.has(o)) { + return objectCache.get(o)! + } + + const payload = o[moveableStr] as ExternalValue | undefined + if (payload === undefined) { + let isPromise = false + const r: Record = {} + objectCache.set(o, r) + + for (const [k, v] of Object.entries(o)) { + r[k] = resolve(v) + isPromise ||= r[k] instanceof Promise + } + + return !isPromise ? r : (async function () { + for (const [k, v] of Object.entries(r)) { + r[k] = await v + } + return r + })() + } + + delete o[moveableStr] + + function resolveWorker(payload: ExternalValue) { + // No value type means we're nested or are a reference + if (!payload.valueType) { + return resolve(payload) + } + + const id = payload.id + if (id !== undefined && resolveCache.has(id)) { + return resolveCache.get(id) + } + + const val = resolveValueType() + + if (id !== undefined) { + resolveCache.set(id, val) + } + + return val + + function resolveValueType() { + if (payload.valueType === 'resource') { + const value = (payload as any).value! + const normalized = normalizeTerraform(value) + // XXX: do not re-hydrate captured values in a bundle + if (typeof normalized === 'object' && !!normalized && 'id' in normalized && 'location' in normalized && 'source' in normalized && 'state' in normalized && 'captured' in normalized) { + normalized.captured = '' + } + + return resolve(normalized) + } + + if (payload.valueType === 'regexp') { + const desc = payload as RegExpValue + + return new RegExp(desc.source, desc.flags) + } + + if (payload.valueType === 'object') { + return loadObject(payload as SerializedObject) + } + + if (payload.valueType === 'reflection') { + return handleReflection(payload.operations!) + } + + if (payload.valueType === 'binding') { + lateBindings.set(payload.id!, { key: payload.key!, value: payload.value! }) + + return {} + } + + if (payload.valueType === 'data-pointer') { + return createPointer((payload as DataPointerValue).hash, (payload as DataPointerValue).storeHash) + } + + if (payload.valueType === 'bound-function') { + return payload.boundTarget!.bind(payload.boundThisArg, ...payload.boundArgs!) + } + + const module = typeof payload.module === 'object' || payload.module.startsWith(pointerPrefix) + ? payload.module + : `${pointerPrefix}${payload.module}` + + const mod = moduleLoader.loadModule(module, payload.location) + const val = mod instanceof Promise + ? mod.then(exports => loadValue(payload, exports)) + : loadValue(payload, mod) + + return val + } + } + + function finalize(payload: any) { + const result = resolveWorker(payload) + objectCache.set(o, result) + + const symbols = resolve(payload.symbols) + if (result instanceof Promise) { + const r2 = result.then(x => addSymbols(x, payload, symbols, preserveId, context)) + // BUG: `resolve(payload.symbols)` could get a stale result if it's dependent on `o` + objectCache.set(o, r2) + + return r2 + } + + return addSymbols(result, payload, symbols, preserveId, context) + } + + const isReference = payload.id !== undefined && payload.valueType === undefined + const resolvedPayload = resolve(isReference ? dataTable[payload.id] : payload) + if (resolvedPayload instanceof Promise) { + return resolvedPayload.then(finalize) + } + + return finalize(resolvedPayload) + } + + const resolved = resolve(value) + + // Fulfill late bindings + for (const [k, v] of lateBindings.entries()) { + const obj = resolveCache.get(k) + if (!obj) { + throw new Error('Expected late bound object to exist') + } + + const val = (typeof v.value === 'number' || typeof v.value === 'string') + ? { [moveableStr]: dataTable[v.value] } + : v.value + + obj[v.key] = resolve(val) + } + + return resolved +} + +let util: typeof import('node:util') | null +function isModule(obj: unknown) { + if (util !== undefined) { + if (util === null) { + return undefined + } + + return util.types.isModuleNamespaceObject(obj) + } + + try { + return util = require('node:util') + } catch { + return util = null + } +} + +export const objectId = Symbol.for('synapse.objectId') + +// The current impl. of symbols fails to handle unregistered symbols +// Well-known symbols provided by v8 e.g. Symbol.iterator are not in +// the global symbol registry, so this statement is true: +// Symbol.for('Symbol.iterator') !== Symbol.iterator +function hydrateSymbol(name: string, globals = globalThis) { + if (name.startsWith('Symbol.')) { + const prop = name.slice('Symbol.'.length) + if ((globals.Symbol as any)[prop]) { + return (globals.Symbol as any)[prop] + } + } + + return Symbol.for(name) +} + +function addSymbols(obj: any, payload: any, symbols?: any, preserveId?: boolean, globals = globalThis) { + if (!obj || (typeof obj !== 'object' && typeof obj !== 'function')) { + return obj + } + + // FIXME: the re-hydration procedure is not symmetric unless we add a proxy here + // (but this might not be problematic) + if (!Object.isExtensible(obj)) { + return obj + } + + if (symbols !== undefined) { + delete symbols['synapse.objectId'] // This symbol is meant to be transient + + const hydratedSymbols = Object.fromEntries( + Object.entries(symbols).map(([k, v]) => [hydrateSymbol(k, globals), v]) + ) + + Object.assign(obj, hydratedSymbols) + } + + if (preserveId) { + return Object.assign(obj, { [objectId]: payload.id }) + } + + // Don't add the payload back unless it's a function or a binding + const isBinding = typeof payload.id === 'string' && payload.id.startsWith('b:') + if (payload.valueType !== 'function' && !isBinding) { + return obj + } + + if (isBinding) { + return Object.assign(obj, { + [Symbol.for("symbolId")]: { + // `id` will be re-generated if this object gets serialized again + } + }) + } + + return Object.assign(obj, { [moveable]: () => ({ ...payload, id: undefined }) }) +} + +// Terraform stores attributes with `_` instead of json so we need to normalize them +const capitalize = (s: string) => s ? s.charAt(0).toUpperCase().concat(s.slice(1)) : s +function normalize(str: string) { + const [first, ...rest] = str.split('_') + + return [first, ...rest.map(capitalize)].join('') +} + +function normalizeTerraform(obj: any): any { + if (typeof obj !== 'object' || !obj) { + return obj + } + + if (Array.isArray(obj)) { + return obj.map(normalizeTerraform) + } + + if (obj[pointerSymbol]) { + return obj + } + + const res: Record = {} + for (const [k, v] of Object.entries(obj)) { + // Don't normalize everything + if (k === moveableStr) { + res[k] = v + } else { + res[normalize(k)] = normalizeTerraform(v) + } + } + + return res +} + diff --git a/src/runtime/modules/services.ts b/src/runtime/modules/services.ts new file mode 100644 index 0000000..c0b7bce --- /dev/null +++ b/src/runtime/modules/services.ts @@ -0,0 +1,297 @@ +//@internal +//# moduleId = synapse:services +//# transform = persist + +import * as core from 'synapse:core' +import * as net from 'synapse:srl/net' +import * as compute from 'synapse:srl/compute' +import { HttpRoute, PathArgs, HttpError, HttpHandler, createFetcher } from 'synapse:http' + +type ServiceRequest = { + [P in keyof T]: T[P] extends (...args: infer U) => any + ? { method: P; args: U } + : never +}[keyof T] + +type Authorizer = (authorization: string, request: ServiceRequest) => Promise | R +type Authorization = () => Promise + +type Promisify = { [P in keyof T]: T[P] extends (...args: infer U) => infer R + ? R extends Promise ? T[P] + : (...args: U) => Promise : T[P] +} + +export type Client = Omit + +/** + * `Service` is distinct from a normal class in that it creates + * a "boundary" between external consumers and its internals. + * + * Normally a class is serialized in its entirety across all consumers, + * which may be problematic when embedded in client-side applications. + */ +export abstract class Service { + private authorizer?: Authorizer + private authorization?: () => Promise | string + protected readonly context!: T + + public constructor() { + this.init() + } + + public addAuthorizer(authorizer: Authorizer) { + this.authorizer = authorizer + } + + public setAuthorization(authorization: () => Promise | string) { + this.authorization = authorization + } + + private init() { + // TODO: recursively do this operation until reaching `Service` as the proto + const proto = Object.getPrototypeOf(this) + const descriptors = Object.getOwnPropertyDescriptors(proto) + const client: Record = {} + + // XXX: a bit hacky. We use `defer` here because instance fields won't be initialized + // until after `init` returns. Normally it would be ok to capture `this` directly but + // in this case we cannot because we are essentially overriding the methods on the + // instance. So we have to capture things indirectly instead. + core.defer(() => { + const service = new compute.HttpService({ + mergeHandlers: true, + auth: this.authorizer ? 'none' : 'native', + }) + + const ctor = proto.constructor + const self: Record = {} + for (const [k, v] of Object.entries(this)) { + if (k in descriptors) continue + self[k] = v + } + + let authz = this.authorization + async function getAuthorization() { + if (!authz) { + throw new Error(`No credentials available`) + } + + return await authz() + } + + client['setAuthorization'] = (fn: () => Promise | string) => void (authz = fn) + + for (const [k, v] of Object.entries(descriptors)) { + if (k !== 'constructor' && typeof v.value === 'function') { + const route = service.addRoute(`POST /__rpc/${k}`, async (req, body: { args: any[] }) => { + const args = body.args + if (self.authorizer) { + const authorization = req.headers.get('authorization') + if (!authorization) { + throw new HttpError('Missing `Authorization` header', { statusCode: 401 }) + } + + const context = await self.authorizer(authorization, { method: k, args }) + const withContext = Object.assign({ context }, self) + Object.setPrototypeOf(withContext, ctor.prototype) // FIXME: can we skip doing this somehow? + + return ctor.prototype[k].call(withContext, ...args) + } + + Object.setPrototypeOf(self, ctor.prototype) // FIXME: can we skip doing this somehow? + + return ctor.prototype[k].call(self, ...args) + }) + + // Backwards compat + service.addRoute(`POST /Default/__rpc/${k}`, async (req, body: { args: any[] }) => { + const args = body.args + if (self.authorizer) { + const authorization = req.headers.get('authorization') + if (!authorization) { + throw new HttpError('Missing `Authorization` header', { statusCode: 401 }) + } + + const context = await self.authorizer(authorization, { method: k, args }) + const withContext = Object.assign({ context }, self) + Object.setPrototypeOf(withContext, ctor.prototype) // FIXME: can we skip doing this somehow? + + return ctor.prototype[k].call(withContext, ...args) + } + + Object.setPrototypeOf(self, ctor.prototype) // FIXME: can we skip doing this somehow? + + return ctor.prototype[k].call(self, ...args) + }) + + if (this.authorizer) { + client[k] = async (...args: any[]) => { + const authorization = await getAuthorization() + const fetcher = createFetcher({ headers: { authorization } }) + + return fetcher.fetch(route, { args } as any) + } + } else { + client[k] = (...args: any[]) => service.callOperation(route, { args } as any) + } + } + } + + Object.assign(this, client) + }) + } +} + +export type Client2 = Promisify> + +export interface ClientConfig { + authorization?: () => Promise | string +} + +function createClientClass(routes: Record): new (config?: ClientConfig) => Client2 { + class Client { + constructor(config?: ClientConfig) { + for (const [k, route] of Object.entries(routes)) { + (this as any)[k] = async (...args: any[]) => { + const authorization = await config?.authorization?.() + const fetcher = createFetcher(authorization ? { headers: { authorization } } : undefined) + + return fetcher.fetch(route, { args } as any) + } + } + } + } + + // TODO: serializing fails for class expressions in return statements + return Client as any +} + +interface ServiceOptions { + readonly domain?: net.HostedZone +} + +export abstract class Service2 { + #authorizer?: Authorizer + protected readonly context!: T + + public constructor(opt?: ServiceOptions) { + this.init(opt) + } + + public addAuthorizer(authorizer: Authorizer) { + this.#authorizer = authorizer + } + + readonly #routes: Record = {} + + private init(opt?: ServiceOptions) { + // TODO: recursively do this operation until reaching `Service` as the proto + const proto = Object.getPrototypeOf(this) + const ctor = proto.constructor + const descriptors = Object.getOwnPropertyDescriptors(proto) + + // `defer` is needed to allow the authorizer to be set + core.defer(() => { + const authorizer = this.#authorizer + const service = new compute.HttpService({ + domain: opt?.domain, + auth: authorizer ? 'none' : 'native', + }) + + for (const [k, v] of Object.entries(descriptors)) { + // Private methods + if (k.startsWith('_')) continue + if (k === 'constructor' || typeof v.value !== 'function') continue + + const route = service.addRoute(`POST /__rpc/${k}`, async (req, body: { args: any[] }) => { + const args = body.args + if (authorizer) { + const authorization = req.headers.get('authorization') + if (!authorization) { + throw new HttpError('Missing `Authorization` header', { statusCode: 401 }) + } + + const context = await authorizer(authorization, { method: k, args } as any) + Object.assign(this, { context }) + + return ctor.prototype[k].call(this, ...args) + } + + return ctor.prototype[k].call(this, ...args) + }) + + this.#routes[k] = route + } + }) + } + + public createClientClass(): ReturnType> { + return createClientClass(this.#routes) + } +} + +type GetContext = T extends Service ? U : never + +function addHttpRoute( + target: T, + route: U, + handler: HttpHandler }> +): HttpRoute<[...PathArgs, string], R> { + throw new Error('TODO') +} + +export function registerSecretProvider(secretType: string, provider: core.SecretProvider) { + new core.SecretProvider2({ + secretType, + getSecret: () => provider.getSecret(), + }) +} + +export const getSecret = core.defineDataSource(async (secretType: string) => { + const client = core.getBackendClient() + const secret = await client.getSecret(secretType) + + return secret +}) + +interface IdentityProviderRegistrationProps { + name?: string + type: string + authenticate: compute.Function, Awaited>> + startAuthentication: compute.Function, Awaited>> +} + +class IdentityProviderRegistration extends core.defineResource({ + create: async (props: IdentityProviderRegistrationProps) => { + const client = core.getBackendClient() + const resp = await client.createIdentityProvider(props) + + return { + providerId: resp.id, + providerType: props.type, + } + }, + delete: async (state) => { + const client = core.getBackendClient() + await client.deleteIdentityProvider(state.providerId) + } +}) {} + +interface IdentityProvider { + name?: string + type: string + authenticate: core.AuthenticateFn + startAuthentication: core.StartAuthenticationFn +} + +export function registerIdentityProvider(idp: IdentityProvider) { + const authenticate = new compute.Function(idp.authenticate) + const startAuthentication = new compute.Function(idp.startAuthentication) + const registration = new IdentityProviderRegistration({ + ...idp, + authenticate, + startAuthentication, + }) + + return registration +} \ No newline at end of file diff --git a/src/runtime/modules/terraform.ts b/src/runtime/modules/terraform.ts new file mode 100644 index 0000000..ed7cd81 --- /dev/null +++ b/src/runtime/modules/terraform.ts @@ -0,0 +1,2042 @@ +//@internal +//# moduleId = synapse:terraform + +module.exports[Symbol.for('moduleIdOverride')] = 'synapse:terraform' + +// --------------------------------------------------------------- // +// ------------------------- TERRAFORM --------------------------- // +// --------------------------------------------------------------- // + +// https://developer.hashicorp.com/terraform/language/syntax/json#expression-mapping + +interface Entity { + readonly name: string + readonly type: string + readonly kind: TerraformElement['kind'] +} + +interface StringLiteral { + readonly value: number | string | object +} + +enum ExpressionKind { + NumberLiteral, + Reference, + PropertyAccess, + ElementAccess, + Call +} + +interface Expression { + readonly kind: ExpressionKind +} + +interface NumberLiteral extends Expression { + readonly kind: ExpressionKind.NumberLiteral + readonly value: number +} + +interface ReferenceExpression extends Expression { + readonly kind: ExpressionKind.Reference + readonly target: Entity +} + +interface CallExpression extends Expression { + readonly kind: ExpressionKind.Call + readonly target: string + //readonly arguments: Expression[] + readonly arguments: any[] +} + +interface PropertyAccessExpression extends Expression { + readonly kind: ExpressionKind.PropertyAccess + readonly expression: Expression + readonly member: string +} + +interface ElementAccessExpression extends Expression { + readonly kind: ExpressionKind.ElementAccess + readonly expression: Expression + readonly element: Expression +} + + + +// const asyncFnProto = Object.getPrototypeOf((async function() {})) + +// FIXME: this cannot be trapped/captured +// const hasOwnProperty = Object.prototype.hasOwnProperty + +function isNonNullObjectLike(o: any) { + return (typeof o === 'object' || typeof o === 'function') && !!o +} + +function isCustomSerializeable(o: object | Function) { + if (moveable in o || isRegExp(o)) { + return true + } + + if (o.constructor && moveable in o.constructor) { + return true + } + + if (expressionSym in o) { + if (!isOriginalProxy(o) || !o.constructor || isGeneratedClassConstructor(o.constructor)) { + return true + } + } + + return false +} + +function isProxy(o: any) { + return !!o && (typeof o === 'object' || typeof o === 'function') && !!Reflect.getOwnPropertyDescriptor(o, moveable2) +} + +function unwrapProxy(o: any): any { + if (isProxy(o)) { + return unwrapProxy(o[unproxy]) + } + + return o +} + +function isJsonSerializeable(o: any, visited = new Set()): boolean { + if (visited.has(o)) { + return false + } + visited.add(o) + + if (typeof o === 'symbol' || typeof o === 'bigint') { + return false + } + if (typeof o === 'function') { + return isCustomSerializeable(o) + } + if (typeof o !== 'object' || o === null || isCustomSerializeable(o)) { // `undefined` is only implicitly serializeable + return true + } + if (Array.isArray(o)) { + return true + } + if (!isObjectOrNullPrototype(Object.getPrototypeOf(o))) { + return false + } + + // This is somewhat lossy as we should only attempt to serialize 'simple' descriptors (value + writable + enumerable + configurable) + for (const desc of Object.values(Object.getOwnPropertyDescriptors(o))) { + if (desc.get || desc.set || !isJsonSerializeable(desc.value, visited)) { + return false + } + } + + return true +} + +const TypedArray = Object.getPrototypeOf(Uint8Array) + +function isObjectOrNullPrototype(proto: any) { + return proto === Object.prototype || proto === null || proto.constructor?.name === 'Object' +} + +export function isGeneratedClassConstructor(o: any) { + if (Object.prototype.hasOwnProperty.call(o, terraformClass)) { + return true + } + + const proto = Object.getPrototypeOf(o) + if (proto && Object.prototype.hasOwnProperty.call(proto, terraformClass)) { + return true + } + + return false +} + +export function isOriginalProxy(o: any) { + return (o as any)[originalSym] +} + +const knownSymbols: symbol[] = [Symbol.iterator] +if (Symbol.dispose) { + knownSymbols.push(Symbol.dispose) +} +if (Symbol.asyncDispose) { + knownSymbols.push(Symbol.asyncDispose) +} + +const permissions = Symbol.for('permissions') +const browserImpl = Symbol.for('browserImpl') +const objectId = Symbol.for('synapse.objectId') + +const unproxy = Symbol.for('unproxy') +const symbolId = Symbol.for('symbolId') // Used to track references when capturing +const serializeableSymbols = new Set([permissions, browserImpl, objectId, ...knownSymbols]) +const reflectionType = Symbol.for('reflectionType') + +function getSymbols(o: any) { + const symbols = Object.getOwnPropertySymbols(o).filter(s => s.description).filter(s => serializeableSymbols.has(s)) + + return symbols.length > 0 ? Object.fromEntries(symbols.map(s => [s.description!, o[s]])) : undefined +} + +function decomposeObject(o: any, keys = new Set(Object.keys(o))): any { + const props = Object.getOwnPropertyNames(o) + const descriptors = Object.getOwnPropertyDescriptors(o) + + // If the object isn't unwrapped then you can get weird cases of self-references + const actualProperties = Object.fromEntries( + props.filter(k => keys.has(k)).map(k => [k, unwrapProxy(o)[k]]) + ) + + // `prototype` is read-only for functions!!! + // THIS DOESN'T WORK CORRECTLY + const includePrototype = (typeof o === 'function' && isNonNullObjectLike(o.prototype) && Object.keys(o.prototype).length > 0) + const actualDescriptors = Object.fromEntries( + Object.entries(descriptors) + .filter(([k]) => (keys.has(k) && !props.includes(k)) || (includePrototype && k === 'prototype')) + ) + + const prototypeSlot = Object.getPrototypeOf(o) + + return { + __constructor: o.constructor?.name !== 'Object' ? o.constructor : undefined, + __prototype: !isObjectOrNullPrototype(prototypeSlot) ? prototypeSlot : undefined, + properties: Object.keys(actualProperties).length > 0 ? actualProperties : undefined, + descriptors: Object.keys(actualDescriptors).length > 0 ? actualDescriptors : undefined, + symbols: getSymbols(o), + } +} + +const pointerSymbol = Symbol.for('synapse.pointer') +type DataPointer = string & { hash: string; [pointerSymbol]: true; resolve(): { hash: string; storeHash: string } } +function isDataPointer(h: string): h is DataPointer { + return typeof h === 'object' && pointerSymbol in h +} + +function renderDataPointer(pointer: DataPointer) { + const { hash, storeHash } = pointer.resolve() + + return renderCallExpression({ + kind: ExpressionKind.Call, + target: 'markpointer', + arguments: [hash, storeHash], + }) +} + +function renderEntity(entity: Entity) { + return `${entity.kind === 'data-source' ? 'data.' : ''}${entity.type}.${entity.name}` +} + +export function isRegExp(o: any): o is RegExp { + return o instanceof RegExp || (typeof o === 'object' && !!o && 'source' in o && Symbol.match in o) +} + +function renderObjectLiteral(obj: any, raw = false): string { + const entries = Object.entries(obj).filter(([_, v]) => v !== undefined) + + return `{${entries.map(([k, v]) => `${renderLiteral(k)} = ${raw ? v : renderLiteral(v)}`).join(', ')}}` +} + +type Ref = { [Symbol.toPrimitive]: () => string } + +interface Context { + readonly moduleId: string + readonly testContext?: any + readonly dataTable: Record +} + +const serializeableResourceClasses = new Set([ + 'Asset', + 'Test', + 'TestSuite', + 'Closure', + 'Custom', + 'CustomData', + + // Legacy + 'Example', + 'ExampleData', +]) + +export function createSerializer( + serialized: State['serialized'] = new Map(), + tables: State['tables'] = new Map() +) { + const moduleIds = new Map() // Used to track reference bindings per-module + + const objectTable = new Map() + const refCounter = new Map() + + const hashes = new Map() + function getHash(ctx: Context) { + const prefix = `${ctx.moduleId}:${ctx.testContext?.id ?? ''}` + if (hashes.has(prefix)) { + return hashes.get(prefix)! + } + + const hash = require('node:crypto') + .createHash('sha1') + .update(prefix) + .digest('hex') + .slice(0, 16) + + hashes.set(prefix, hash) + + return hash + } + + const idTable = new Map() + function generateId(ctx: Context) { + const prefix = getHash(ctx) + const count = (idTable.get(prefix) ?? 0) + 1 + idTable.set(prefix, count) + + return `${prefix}_${count}` + } + + const depsStack: Set[] = [] + function addDep(id: string, name: string, ctx: Context) { + if (depsStack.length > 0) { + depsStack[depsStack.length - 1].add(id) + } else { + ctx.dataTable[id] = `\${local.${name}}` + } + } + + // Used for debugging + const serializeStack: any[] = [] + class SerializationError extends Error { + public readonly serializeStack = [...serializeStack] + } + + function getTable(ctx: Context) { + if (tables.has(ctx)) { + return tables.get(ctx)!.ref + } + + const id = generateId(ctx) + const name = `d_${id}` + const ref = renderEntity({ type: 'local', kind: 'resource', name }) + tables.set(ctx, { name, ref }) + + return ref + } + + function incRefCount(o: any, ctx: Context) { + if (!objectTable.has(o)) { + throw new Error(`Object was never registered: ${o}`) + } + + const { id, name, ref } = objectTable.get(o)! + addDep(id, name, ctx) + + // Self-reference, we have to inline a reference here instead of using the ref counter + // + // This means the current object is no longer directly serializeable without the + // object being self-referenced included in its serialization + // if (scopes.includes(id)) { + // return { [`@@${moveable.description!}`]: { id, valueType: 'reference' } } + // } + + const refCount = refCounter.get(name)! + refCounter.set(name, refCount + 1) + + return ref + } + + function getId(obj: any, ctx: Context) { + if (objectTable.has(obj)) { + throw new Error(`Unexpected duplicate "getId" call`) + } + + // IMPORTANT: the `id` is determined when the object is first serialized + // as opposed to when the object is created (the ideal). In most situations + // this works fine, though it's still possible to see unexpected changes. + + // const id = objectTable.size + const id = generateId(ctx) + const name = `o_${id}` + const ref = getReference(id, ctx) + objectTable.set(obj, { id, name, ref, ctx }) + refCounter.set(name, 1) + addDep(id, name, ctx) + + return { id, ref } + } + + function getReference(id: number | string, ctx: Context, lateBound = false) { + function resolve(type?: string) { + const { obj, name, refCount, isMoveable, idOverride } = serialized.get(id)! + // TODO: just never inline? this is buggy + // or we can resolve all call expressions before synth + if (refCount <= 1 && !isMoveable) { + return obj[Symbol.toPrimitive](type) + } + + // The latebound ref is a bare id and should not be treated as a config object + if (lateBound && type === 'object') { + return undefined + } + + if (lateBound) { + return idOverride ?? id + } + + if (!isMoveable) { + return `local.${name}` + } + + return renderLiteral({ [`@@${moveable.description!}`]: { id: idOverride ?? id } }) + } + + return { [Symbol.toPrimitive]: resolve } + } + + function getLateBoundRef(o: any, ctx: Context) { + if (!objectTable.has(o)) { + throw new Error(`Object was never registered: ${o}`) + } + + const { id } = objectTable.get(o)! + + return getReference(id, ctx, true) + } + + class DataClass { + static [terraformClass] = true + constructor(obj: any, data: { encoded: any, isMoveable: boolean, deps: string[], idOverride?: string }) { + if (!objectTable.get(obj)) { + throw new Error(`Object was never registered: ${data}`) + } + + const { id, name, ctx } = objectTable.get(obj)! + + if (serialized.has(id)) { + throw new Error(`Object was created more than once`) + } + + const inlineVal = data.isMoveable + ? { [`@@${moveable.description!}`]: data.encoded } + : data.encoded + + const state = { + name, + type: 'local', + state: data.encoded, + kind: 'resource' as const, + ...ctx, + } + + const entity = createEntityProxy(this, state, {}) + const proxy = new Proxy(entity, { + get: (_, prop) => { + if (prop === Symbol.toPrimitive) { + return (type?: string) => type === 'object' ? inlineVal : renderLiteral(inlineVal) + } + + return entity[prop] + } + }) + + serialized.set(id, { + ctx, + name, + obj: proxy, + deps: data.deps, + isMoveable: data.isMoveable, + idOverride: data.idOverride, + get refCount() { + return refCounter.get(name) ?? 1 + } + }) + + return proxy + } + } + + function withContext(ctx: Context) { + function peekReflectionType(obj: any) { + if (!obj || (typeof obj !== 'object' && typeof obj !== 'function')) { + return + } + + return obj[reflectionType] + } + + function serializeObject(obj: any): any { + const unproxied = peekReflectionType(obj) === 'global' ? obj : unwrapProxy(obj) + if (objectTable.has(unproxied)) { + return incRefCount(unproxied, ctx) + } + + const { id, ref } = getId(unproxied, ctx) + + depsStack.push(new Set()) + serializeStack.push(obj) + + let isMoveable = true + let idOverride: string | undefined + new DataClass(unproxied, { + encoded: serializeData(), + isMoveable, + idOverride, + deps: Array.from(depsStack.pop()!), + }) + + serializeStack.pop() + + return ref + + function serializeData(): any { + if (Array.isArray(obj)) { + isMoveable = false + + return obj.map(serialize) + } + + if (isRegExp(obj)) { + return { + id, + valueType: 'regexp', + source: obj.source, + flags: obj.flags, + } + } + + if (isElement(obj) && obj[internalState].type === 'synapse_resource') { + const subtype = obj[internalState].state.type + + if (serializeableResourceClasses.has(subtype)) { + const exp = (obj as any)[expressionSym] + if (exp.kind === ExpressionKind.Reference) { + const val = (obj as any)[synapseOutput] + + // Dervived classes of `ConstructClass` + if (obj.constructor && !isGeneratedClassConstructor(obj.constructor) && !isObjectOrNullPrototype(Object.getPrototypeOf(obj))) { + return { + id, + properties: val, // FIXME: doesn't handle extra props + valueType: 'object', + constructor: serialize(obj.constructor), + } + } + + isMoveable = false + + return val + } + } + } + + // TODO: maybe fail on any reflection ops over `synapse:*` + // the built-in modules aren't serializeable although maybe they should be? + // right now it only causes problems if a different version of the runtime is used + // so another solution is to automatically grab the correct package version + + if (expressionSym in obj) { + const exp = (obj as any)[expressionSym] + if (exp.kind == ExpressionKind.Call) { // Maybe not needed + isMoveable = false + + return obj + } + + if (exp.kind === ExpressionKind.Reference) { + const entity = exp.target as InternalState + if (entity.kind === 'provider') { + // // This is a super class + // if (obj.constructor && !Object.prototype.hasOwnProperty(terraformClass)) { + + // } + + // FIXME: generalize this pattern instead of hard-coding + if (entity.type === 'aws') { + return obj.roleArn ? { roleArn: obj.roleArn } : {} + } + + console.log(`Unable to serialize entity of kind "provider": ${entity.type}.${entity.name}`) + isMoveable = false + + // We'll dump the provider's config anyway... + return {} // { ...obj } + } + } + + // XXX: resource fields from Terraform need to be mapped + return { + id, + valueType: 'resource', + value: obj, + } + } + + const desc = resolveMoveableDescription(obj) + if (desc?.type === 'direct') { + return serializeObjectLiteral({ ...desc.data, id }) + } + + if (typeof obj.constructor === 'function') { + const ctorDesc = resolveMoveableDescription(obj.constructor) + + if (ctorDesc?.type === 'direct') { + return serializeFullObject(id, obj) + } + } + + // This is a best-effort serialization... + if (desc?.type === 'reflection') { + return serializeObjectLiteral({ ...desc.data, id }) + } + + if (isJsonSerializeable(obj)) { + if (symbolId in obj) { + const desc = obj[symbolId] + const boundId = getBoundSymbolId(id, desc) + idOverride = boundId + + if (desc.lateBound) { + const key = Object.keys(obj)[0] + const unproxied = peekReflectionType(obj[key]) === 'global' + ? obj[key] + : unwrapProxy(obj[key]) + + serialize(unproxied) + const ref = getLateBoundRef(unproxied, ctx) + + return { + id: boundId, + valueType: 'binding', + key, + value: ref, + } + } + + return serializeFullObject(boundId, obj) + } + + return serializeFullObject(id, obj) + + // isMoveable = false + + // return serializeObjectLiteral(obj) + } + + if (obj instanceof Uint8Array) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'Uint8Array' }, + { type: 'construct', args: [Array.from(obj.values())] }, + ] + } + } + + if (obj instanceof Map) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'Map' }, + { type: 'construct', args: [Array.from(obj.entries()).map(serialize)] }, + ] + } + } + + // Serialized WeakMaps and WeakSets will drop all entries + if (obj instanceof WeakMap) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'WeakMap' }, + { type: 'construct', args: [] }, + ] + } + } + + if (obj instanceof WeakSet) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'WeakSet' }, + { type: 'construct', args: [] }, + ] + } + } + + if (obj instanceof Set) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'Set' }, + { type: 'construct', args: [Array.from(obj.values())] }, + ] + } + } + + if (typeof obj === 'function') { + if (obj.name === 'Object' && Object.keys(Object).every(k => Object.prototype.hasOwnProperty.call(obj, k))) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'Object' } + ] + } + } + + if (obj === Uint8Array) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'Uint8Array' } + ] + } + } + + if (obj === TypedArray) { + return { + id, + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'Object' }, + { type: 'get', property: 'getPrototypeOf' }, + { type: 'apply', args: [serialize(Uint8Array)] } + ] + } + } + + throw new SerializationError(`Failed to serialize function: ${obj.toString()}`) + } + + return serializeFullObject(id, obj) + } + } + + function resolveMoveableDescription(obj: any) { + const symbols = getSymbols(obj) + const direct = typeof obj[moveable] === 'function' ? obj[moveable]() : undefined + const reflection = typeof obj[moveable2] === 'function' ? obj[moveable2]() : undefined + + if ((!direct && reflection)) { + const desc = symbols ? { symbols, ...reflection } : reflection + + return { type: 'reflection' as const, data: desc } + } + + if (direct) { + const desc = symbols ? { symbols, ...direct } : direct + + return { type: 'direct' as const, data: desc } + } + } + + function serializeObjectLiteral(obj: any) { + return obj ? Object.fromEntries( + Object.entries(obj).map(([k, v]) => [k, serialize(v)]) + ) : undefined + } + + function serializeFullObject(id: number | string, obj: any) { + const decomposed = decomposeObject(obj) + + // Note: `prototype` refers to [[prototype]] here + const ctor = decomposed.__constructor ?? decomposed.__prototype?.constructor + if (ctor && moveable in ctor) { + delete decomposed['__prototype'] + ;(decomposed as any).__constructor = ctor + } + + const desc = { + valueType: 'object', + ...decomposed + } + + const finalDesc = (serializeSym in obj && typeof obj[serializeSym] === 'function') ? obj[serializeSym](desc) : desc + + return { + id, + valueType: finalDesc.valueType, + prototype: serialize(finalDesc.__prototype), + constructor: serialize(finalDesc.__constructor), + properties: serializeObjectLiteral(finalDesc.properties), + descriptors: serializeObjectLiteral(finalDesc.descriptors), + __privateFields: serializeObjectLiteral(finalDesc.__privateFields), + symbols: serializeObjectLiteral(finalDesc.symbols), + } + } + + function getBoundSymbolId(id: number | string, desc: { id?: number; origin?: string }) { + // This object was rehydrated, use the reference id + if (!desc.id || !desc.origin) { + return `b:${id}` + } + + if (!moduleIds.has(desc.origin)) { + moduleIds.set(desc.origin, moduleIds.size) + } + + const moduleId = moduleIds.get(desc.origin)! + + return 'b:' + (desc.id << 8) + (moduleId & 0xFF) // TODO: make this better + } + + function serializeSymbol(obj: symbol) { + // COPY-PASTED + if (objectTable.has(obj)) { + return incRefCount(obj, ctx) + } + + const desc = obj.description + if (!desc) { + throw new SerializationError(`Unable to capture symbol without a description: ${obj.toString()}`) + } + + const { id, ref } = getId(obj, ctx) + + const thisArg = { + valueType: 'reflection', + operations: [ + { type: 'global' }, + { type: 'get', property: 'Symbol' } + ] + } + + depsStack.push(new Set()) + + new DataClass(obj, { + encoded: { + id, + valueType: 'reflection', + operations: [ + ...thisArg.operations, + { type: 'get', property: 'for' }, + { type: 'apply', args: [desc], thisArg } + ] + }, + isMoveable: true, + deps: Array.from(depsStack.pop()!), + }) + + return ref + } + + function serialize(obj: any): any { + if (obj === null) { + return obj + } + + if (isDataPointer(obj)) { + return `\${${renderDataPointer(obj)}}` + } + + switch (typeof obj) { + case 'object': + case 'function': + return serializeObject(obj) + + case 'symbol': + return serializeSymbol(obj) + + default: + return obj + } + } + + return { serialize, getTable: () => getTable(ctx) } + } + + return { + withContext, + serialized, + tables, + } +} + +interface Serializer { + serialize: (obj: any) => string + getTable: () => string +} + +function renderLiteral(obj: any, isEncoding = false, serializer?: Serializer): string { + if (Array.isArray(obj)) { + return `[${obj.map(x => renderLiteral(x, isEncoding, serializer)).join(', ')}]` + } + + if (typeof obj === 'string') { + // This is awful... + const pattern = /([^]*[^$]?)\$\{(.*)\}([^]*)/g + let res: string = '' + let match: RegExpExecArray | null + while (match = pattern.exec(obj)) { + res += JSON.stringify(match[1]).slice(1, -1) + `\${${match[2]}}` + JSON.stringify(match[3]).slice(1, -1) + } + if (res) return '"' + res + '"' + + return JSON.stringify(obj) + } + + if (typeof obj === 'number') { + return String(obj) + } + + if (typeof obj === 'boolean') { + return obj ? 'true' : 'false' + } + + if (obj === null) { + return 'null' + } + + // Not correct + if (obj === undefined) { + return 'null' + } + + if (isRegExp(obj)) { + return JSON.stringify(`/${obj.source}/`) + } + + if (isDataPointer(obj)) { + return renderDataPointer(obj) + } + + if (serializer && isEncoding) { + return serializer.serialize(obj) + } + + if (expressionSym in obj) { + return render((obj as any)[expressionSym]) + } + + if (Object.prototype.hasOwnProperty.call(obj, Symbol.toPrimitive)) { + return obj[Symbol.toPrimitive]('string') + } + + if (typeof obj === 'function') { + throw new Error(`Unable to render function: ${obj.toString()}`) + } + + if (typeof obj === 'symbol') { + throw new Error(`Unable to render symbol: ${obj.toString()}`) + } + + return renderObjectLiteral(obj) +} + +function renderCallExpression(expression: CallExpression, serializer?: Serializer) { + if (expression.target === 'serialize') { + const captured = renderLiteral(expression.arguments[0], true, serializer) + + return renderObjectLiteral({ + captured, + table: serializer?.getTable(), + __isDeduped: true, + }, true) + } + + const target = expression.target === 'encoderesource' ? 'jsonencode' : expression.target + const isEncoding = expression.target === 'encoderesource' + const args = expression.arguments.map(x => renderLiteral(x, isEncoding, serializer)) + + return `${target}(${args.join(', ')})` +} + +function render(expression: Expression, serializer?: Serializer): string { + switch (expression.kind) { + case ExpressionKind.Reference: + return renderEntity((expression as ReferenceExpression).target) + case ExpressionKind.PropertyAccess: + return `${render((expression as PropertyAccessExpression).expression)}.${(expression as PropertyAccessExpression).member}` + case ExpressionKind.ElementAccess: + return `${render((expression as ElementAccessExpression).expression)}[${render((expression as ElementAccessExpression).element)}]` + case ExpressionKind.NumberLiteral: + return renderLiteral((expression as NumberLiteral).value) + case ExpressionKind.Call: + return renderCallExpression(expression as CallExpression, serializer) + } +} + +function createEntityProxy(original: any, state: InternalState, mappings?: Record) { + return createProxy({ + kind: ExpressionKind.Reference, + target: state, + } as any, state, mappings, original,) +} + +export const internalState = Symbol.for('internalState') +const expressionSym = Symbol.for('expression') +const mappingsSym = Symbol.for('mappings') +export const moveable = Symbol.for('__moveable__') +const moveable2 = Symbol.for('__moveable__2') + +export const originalSym = Symbol.for('original') +const serializeSym = Symbol.for('serialize') + +interface TerraformElementBase { + readonly name: string + readonly type: string + readonly state: any + readonly module: string + readonly mappings?: Record +} + +interface TerraformResourceElement extends TerraformElementBase { + readonly kind: 'resource' +} + +interface TerraformDataSourceElement extends TerraformElementBase { + readonly kind: 'data-source' +} + +interface TerraformLocalElement extends TerraformElementBase { + readonly kind: 'local' +} + +interface TerraformProviderElement extends TerraformElementBase{ + readonly kind: 'provider' + readonly source: string + readonly version: string +} + +export type TerraformElement = + | TerraformProviderElement + | TerraformResourceElement + | TerraformDataSourceElement + | TerraformLocalElement + +function mapKey(key: string, mappings: Record) { + if (!mappings) { + return key + } + + if (key in mappings) { + const map = mappings[key] + const val = typeof map === 'object' ? map[''] : map + + return val === '' ? toSnakeCase(key) : val + } + + return key +} + +function createTfExpression() { + const terraformExpression = function () {} + ;(terraformExpression as any)[internalState] = true + ;(terraformExpression as any)[expressionSym] = true + ;(terraformExpression as any)[mappingsSym] = true + + return terraformExpression +} + +export function createProxy( + expression: Expression, + state: InternalState | undefined, + mappings: Record = {}, + original?: any, +): any { + const serializer = state?.['__serializer'] + + // We cache all created proxies to ensure referential equality + const proxies = new Map() + function createInnerProxy(key: PropertyKey, expression: Expression) { + if (proxies.has(key)) { + return proxies.get(key) + } + + const isPropertyAccess = expression.kind === ExpressionKind.PropertyAccess && typeof key === 'string' + const childMappings = isPropertyAccess ? mappings[key] : mappings + const inner = createProxy(expression, state, childMappings) + proxies.set(key, inner) + + return inner + } + + // Remove this entity from the test context if referenced outside of the context + function checkContext() { + if (state?.testContext) { + const refCtx = getTestContext() + if (!refCtx) { + delete state['testContext'] + } + } + } + + const target = original ? original : createTfExpression() + + return new Proxy(target, { + get: (target, prop, receiver) => { + if (prop === internalState) { + return state + } + + if (prop === expressionSym) { + return expression + } + + if (prop === mappingsSym) { + return mappings + } + + if (prop === originalSym) { + return original + } + + if (prop === 'toString') { + checkContext() + + return () => `\${${render(expression, serializer)}}` + } + + if (prop === 'toJSON') { + return () => `\${${render(expression, serializer)}}` + } + + // if (prop === 'slice') { + // // call `substr` + // return () => `\${${render(expression)}}` + // } + + if (prop === Symbol.toPrimitive) { + return () => `\${${render(expression, serializer)}}` + } + + if (original && Reflect.has(original, prop)) { + return Reflect.get(original, prop, receiver) + } + + if (typeof prop === 'symbol') { + return target?.[prop] ?? (state as any)?.[prop] + } + + // Terraform doesn't allow expressions on providers + // But we support accessing the initial configuration of a provider + if (state?.kind === 'provider') { + return undefined + } + + const exp = !isNaN(Number(prop)) + ? { + kind: ExpressionKind.ElementAccess, + expression, + element: { + kind: ExpressionKind.NumberLiteral, + value: Number(prop), + } + } + : { + kind: ExpressionKind.PropertyAccess, + expression, + member: mapKey(prop, mappings), + } + + return createInnerProxy(prop, exp) + }, + apply: (target, thisArg, argArray) => { + const args = thisArg !== undefined + ? [thisArg, ...argArray] + : argArray + + return createCallExpression((expression as PropertyAccessExpression).member, args, state) + }, + has: (target, prop) => { + if (prop === internalState || prop === expressionSym || prop === mappingsSym || prop === originalSym) { + return true + } + + // if (typeof prop === 'string' && mappings[prop]) { + // return true + // } + + if (!original) { + return false + } + + return Reflect.has(original, prop) + }, + + // This would only work with statically known props + // ownKeys: (target) => { + // return Array.from(new Set([ + // internalState, + // expressionSym, + // mappingsSym, + // originalSym, + // ...(Object.keys(mappings)), + // ...(original ? Reflect.ownKeys(original) : []) + // ])) + // } + // getOwnPropertyDescriptor: (target, prop) => { + // if (prop === internalState || prop === expressionSym || prop === mappingsSym || prop === originalSym) { + // return { value: true, configurable: true, enumerable: true } + // } + + // if (original) { + // return Object.getOwnPropertyDescriptor(original, prop) + // } + // }, + }) +} + +const createCallExpression = (name: string, args: any[], state?: any) => createProxy({ + kind: ExpressionKind.Call, + target: name, + arguments: args, // FIXME: don't use jsonencode for this +} as any, state, ((typeof args[0] === 'object' || typeof args[0] === 'function') && !!args[0] && mappingsSym in args[0]) ? (args[0] as any)[mappingsSym] : undefined) + + +export interface TfJson { + readonly '//'?: Extensions + readonly provider: Record + readonly resource: Record> + readonly data: Record> + readonly terraform: { backend?: Record, 'required_providers': Record } + readonly moved: { from: string, to: string }[] + readonly locals: Record +} + +function isDefaultProviderName(name: string, type: string) { + return name === '#default' || name === 'default' || name === type +} + +function isDefaultProvider(element: TerraformElement) { + return isDefaultProviderName(element.name, element.type) +} + +// terraform: 'terraform.io/builtin/terraform', + +function computeSizeTree(o: any): any { + if (typeof o === 'string' || typeof o === 'number' || typeof o === 'boolean' || o === null) { + return String(o).length + } + + if (typeof o !== 'object' && typeof o !== 'function') { + return 0 + } + + if (Array.isArray(o)) { + return o.map(computeSizeTree) + } + + if (expressionSym in o) { + return computeSizeTree(JSON.parse(`"${String(o)}"`)) + } + + function getSize(o: any): number { + if (typeof o === 'number') { + return o + } + + if (Array.isArray(o)) { + return o.reduce((a, b) => a + getSize(b), 0) + } + + return o['__size'] + } + + if (typeof o === 'object') { + let totalSize = 0 + const res: Record = {} + for (const [k, v] of Object.entries(o)) { + const size = computeSizeTree(v) + totalSize += getSize(size) + res[k] = size + } + + res['__size'] = totalSize + + return res + } + + throw new Error(`Bad type: ${JSON.stringify(o)}`) +} + + +function escapeRegExp(pattern: string) { + return pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} + +export interface HookContext { + moveResource(from: string, to: TerraformResourceElement): void +} + +export type SynthHook = (element: TerraformElement, context: HookContext) => TerraformElement | void + +export interface Symbol { + name: string + line: number // 0-indexed + column: number // 0-indexed + fileName: string +} + +export interface ExecutionScope { + isNewExpression?: boolean + callSite?: Symbol + assignment?: Symbol + namespace?: Symbol[] +} + +export interface TerraformSourceMap { + symbols: Symbol[] + resources: Record +} + +export interface PackageInfo { + readonly type: 'npm' | 'cspm' | 'synapse-provider' | 'file' | 'jsr' | 'synapse-tool' | 'spr' | 'github' + readonly name: string + readonly version: string + readonly packageHash?: string // Only relevant for `synapse` + readonly os?: string[] + readonly cpu?: string[] + readonly resolved?: { + readonly url: string + readonly integrity?: string + } +} + +export interface TerraformPackageManifest { + roots: Record + packages: Record + dependencies: Record> +} + +interface ResolvedDependency { + readonly package: number + readonly versionConstraint: string +} + +// These are embedded as comments into the Terraform format +interface Extensions { + deployTarget?: string + secrets?: Record // TODO: make secrets into a proper data source + sourceMap?: TerraformSourceMap +} + +function initTfJson(): TfJson { + return { + '//': {}, + provider: {}, + resource: {}, + data: {}, + terraform: { required_providers: {} }, + moved: [], + locals: {}, + } +} + +// MUTATES +function deleteEmptyKeys(obj: T): Partial { + for (const [k, v] of Object.entries(obj)) { + if (v && typeof v === 'object' && Object.keys(v).length === 0) { + delete (obj as any)[k] + } + } + + return obj +} + +function getModuleName(element: InternalState) { + const moduleId = element.module + const testSuiteId = element.testContext?.id + + return `${moduleId}${testSuiteId !== undefined ? `#test-suite=${testSuiteId}` : ''}` +} + +function strcmp(a: string, b: string) { + return a < b ? -1 : a > b ? 1 : 0 +} + +interface SourceMapper { + addSymbols(resourceName: string, scopes: ExecutionScope[]): void + getSourceMap(): TerraformSourceMap +} + +function emitTerraformJson( + state: State, + sourceMapper: SourceMapper, + hooks: { before?: SynthHook[] } = {}, +): { main: TfJson } { + const tfJson: TfJson = { + ...initTfJson(), + } + + const synthedSizes: Record = {} + + const hookContext: HookContext = { + moveResource(from, to) { + const type = to.type + tfJson.moved.push({ + from: type + '.' + from, + to: type + '.' + to.name, + }) + }, + } + + function before(element: TerraformElement) { + if (!hooks.before) { + return element + } + + return hooks.before.reduce((a, b) => b(a, hookContext) ?? a, element) + } + + const sortedResources = Array.from(state.registered.entries()).sort((a, b) => strcmp(a[0], b[0])) + for (const [k, v] of sortedResources) { + const element = before(v[internalState] as TerraformElement) + element.state['module_name'] = getModuleName(element) + + const mappings = v[mappingsSym] + const synthed = synth(element.state, mappings, element) + const name = element.name + + // Only add symbols for resources for now. Providers/data nodes aren't as useful to show + // without resolving them i.e. calling the provider + const scopes = (element as InternalState).scopes + if (scopes && element.kind === 'resource') { + sourceMapper.addSymbols(`${element.type}.${element.name}`, scopes) + } + + if (element.kind === 'provider') { + if (!isDefaultProvider(element)) { + synthed['alias'] = element.name + } + + tfJson.provider[element.type] ??= [] + tfJson.provider[element.type].push(synthed) + + if (!tfJson.terraform.required_providers[element.type]) { + tfJson.terraform.required_providers[element.type] = { + source: element.source, + // version? + } + } + } + if (element.kind === 'resource') { + const resources = tfJson.resource[element.type] ??= {} + resources[name] = synthed + } + if (element.kind === 'data-source') { + const sources = tfJson.data[element.type] ??= {} + sources[name] = synthed + } + } + + const pruned = new Set() + const sortedSerialized = Array.from(state.serialized.entries()).sort((a, b) => strcmp(a[0] as string, b[0] as string)) + for (const [id, v] of sortedSerialized) { + if (v.refCount > 1 || v.isMoveable) { + const element = v.obj[internalState] as TerraformElement + tfJson.locals[v.name] = synth(element.state, undefined, element) + } else { + pruned.add(id as string) + } + } + + if (state.tables) { + // Merge all transitive data segments + function merge(ctx: Context, seen = new Set()): Record { + const m: Record = {} + function visit(id: string) { + if (seen.has(id)) { + return + } + + seen.add(id) + const o = state.serialized.get(id)! + for (const d of o.deps) { + visit(d) + } + if (!pruned.has(id)) { + m[o.idOverride ?? id] = `\${local.${o.name}}` + } + } + + for (const [k, v] of Object.entries(ctx.dataTable)) { + visit(k) + } + + return Object.fromEntries(Object.entries(m).sort(([a, b]) => strcmp(a[0], b[0]))) + } + const sortedTable = Array.from(state.tables.entries()).sort((a, b) => strcmp(a[1].name, b[1].name)) + for (const [t, { name }] of sortedTable) { + tfJson.locals[name] = merge(t) + } + } + + for (const [k, v] of state.backends.entries()) { + const backend = tfJson.terraform.backend ??= {} + backend[k] = synth(v, {}) + } + + if (state.secrets.size > 0) { + tfJson['//']!.secrets = Object.fromEntries(state.secrets.entries()) + } + + tfJson['//']!.sourceMap = sourceMapper.getSourceMap() + + deleteEmptyKeys(tfJson) + + return { main: tfJson } +} + +const uncapitalize = (s: string) => s ? s.charAt(0).toLowerCase().concat(s.slice(1)) : s +function toSnakeCase(str: string) { + const pattern = /[A-Z]/g + const parts: string[] = [] + + let lastIndex = 0 + let match: RegExpExecArray | null + while (match = pattern.exec(str)) { + parts.push(str.slice(lastIndex, match.index)) + lastIndex = match.index + } + + if (lastIndex !== str.length) { + parts.push(str.slice(lastIndex, str.length)) + } + + return parts.map(uncapitalize).join('_') +} + +export function isElement(o: unknown): o is { [internalState]: TerraformElement } { + return ( + !!o && + (typeof o === 'object' || typeof o === 'function') && + internalState in o && + typeof o[internalState] === 'object' + ) +} + +const getElementKey = (element: Entity & { module?: string }) => `${element.module ?? 'global'}_${element.kind}_${element.type}_${element.name}` + +// INTERNAL ONLY +export function overrideId(o: any, id: string) { + if (!isElement(o)) { + throw new Error(`Object is not a terraform element`) + } + + const s = o[internalState] + const key = getElementKey(s) + const r = getState().registered.get(key) + if (!r) { + throw new Error(`Resource not found within current state: ${s.type}.${s.name}`) + } + + getState().registered.delete(key) + Object.assign(s, { name: id }) + getState().registered.set(getElementKey(s), r) + + return o +} + +function synth(obj: any, mappings?: Record, parent?: TerraformElement): any { + if (isElement(obj)) { + return obj.toString() + } + + if (Array.isArray(obj)) { + return obj.map(x => synth(x, mappings)) + } + + if (obj instanceof RegExp) { + return `/${obj.source}/` + } + + if (typeof obj !== 'object' || !obj) { + return obj + } + + if (isDataPointer(obj)) { + return `\${${renderDataPointer(obj)}}` + } + + if (Object.prototype.hasOwnProperty.call(obj, Symbol.toPrimitive)) { + // Try serializing the target as a ref first before falling back to a literal + const res = obj[Symbol.toPrimitive]('object') + if (res === undefined) { + return obj[Symbol.toPrimitive]('string') + } + + if (typeof res === 'string') { + return `\${${res}}` + } + + return synth(res) + } + + const res: Record = {} + for (const [k, v] of Object.entries(obj)) { + if (k === 'module_name') { + res[k] = v + } else if (k === 'lifecycle' && Array.isArray(v)) { + res[k] = v.map(x => synthLifecycle(x, mappings ?? {})) + } else if (k === 'depends_on' && Array.isArray(v) && parent) { + res[k] = v.filter(x => expressionSym in x).map(x => render(x[expressionSym])) + } else if (k === 'provider' && (parent?.kind === 'resource' || parent?.kind === 'data-source')) { + if (!isElement(v)) { + throw new Error(`Expected element value for key: ${k}`) + } + + const element = v[internalState] + if (!isDefaultProvider(element)) { + res[k] = `${element.type}.${element.name}` + } + } else { + res[mappings ? mapKey(k, mappings) : k] = synth(v, mappings?.[k]) + } + } + + return res +} + +function synthLifecycle(obj: any, mappings: Record) { + const ignoreChanges = Array.isArray(obj['ignore_changes']) + ? obj['ignore_changes'].map(k => mapKey(k, mappings)) + : undefined + + const replaceTriggeredBy = Array.isArray(obj['replace_triggered_by']) + ? obj['replace_triggered_by'].filter(x => expressionSym in x).map(x => render(x[expressionSym])) + : undefined + + const hook = Array.isArray(obj['hook']) + ? obj['hook'].map(x => { + return { + kind: x.kind, + input: x.input.toString(), + handler: x.handler.toString(), + } + }) + : undefined + + return { + ...obj, + hook, + ignore_changes: ignoreChanges, + replace_triggered_by: replaceTriggeredBy, + } +} + +export function updateResourceConfiguration(obj: T, fn: (obj: T) => void): void { + updateResourceConfigurationWorker(obj, fn) +} + +// We recursively search the object, applying the changes to any resource found +// Does not apply to resources within arrays +function updateResourceConfigurationWorker(obj: any, fn: (obj: any) => void, visited = new Set()) { + if ((typeof obj !== 'function' && typeof obj !== 'object') || !obj || Array.isArray(obj) || visited.has(obj)) { + return + } + + visited.add(obj) + if (internalState in obj && typeof (obj as any)[internalState] === 'object') { + fn(obj[internalState].state) + } + + for (const v of Object.values(obj)) { + updateResourceConfigurationWorker(v, fn, visited) + } +} + +export function getAllResources(obj: any, keepExpressions = false, visited = new Set()): any[] { + if ((typeof obj !== 'function' && typeof obj !== 'object') || !obj || visited.has(obj)) { + return [] + } + + visited.add(obj) + if (Array.isArray(obj)) { + return obj.map(x => getAllResources(x, keepExpressions, visited)).reduce((a, b) => a.concat(b), []) + } + + if (internalState in obj && typeof (obj as any)[internalState] === 'object') { + if (!keepExpressions && !(obj as any)[originalSym]) { + return [] + } + + return [obj] + } + + return getAllResources(Array.from(Object.values(obj)), keepExpressions, visited) +} + +function getTestContext() { + const contexts = getProviders() + const testSuite = contexts['test-suite']?.[0] + if (testSuite && typeof testSuite.id !== 'string' && typeof testSuite.id !== 'number') { + throw new Error(`Test context is missing an "id" field`) + } + + return testSuite as { id: string | number } +} + +function getTerraformProviders(): Record { + return Object.fromEntries( + Object.entries(getProviders()).map( + ([k, v]) => [k, v.filter(isElement).filter(x => x[internalState].kind === 'provider')] + ) + ) +} + +function getProviderForElement(element: { name: string, type: string }) { + const allProviders = getTerraformProviders() + const elementProviderType = element.type.split('_').shift()! + + const slotted = allProviders[elementProviderType] + const matched = slotted?.[0] + if (!matched) { + // Automatic init?? + // console.log('current providers',slotted?.map(p => p[internalState].name + ' ' + `(${p[internalState].type})`)) + console.log('current providers (unfiltered)', getProviders()) + + throw new Error(`No provider found for element: ${element.name} (${element.type})`) + } + + return matched +} + +interface InternalState { + module?: string + name: string + type: string + kind: TerraformElement['kind'] + mappings?: Record + version?: string + source?: string + state: any // This is the resource state + scopes?: { callSite?: Symbol; assignment?: Symbol; namespace?: Symbol[] }[] + testContext?: any + __serializer?: Serializer +} + +export const peekNameSym = Symbol('peekName') + +export function createTerraformClass( + type: string, + kind: TerraformElement['kind'], + mappings?: Record, + version?: string, + source?: string, +): new (...args: any[]) => T { + const isSynapse = type === 'synapse_resource' + + const c = class { + static [terraformClass] = true + static [peekNameSym] = () => `${type}.${peekName(type, kind, getScopedId(true))}` + + constructor(...args: any[]) { + const props = (typeof args[args.length - 1] !== 'string' ? args[args.length - 1] : args[args.length - 2]) ?? {} + const csType = isSynapse ? props['type'] : undefined + const name = typeof args[args.length - 1] === 'string' + ? args[args.length - 1] + : generateName(type, kind, getScopedId(), csType) + + if (kind === 'resource' || kind === 'data-source') { + props['provider'] = getProviderForElement({ name, type }) + } else if (kind === 'provider') { + Object.assign(this, props) + // Object.defineProperty(this, Symbol.for('context'), { + // get: () => ({ [type]: proxy }) + // }) + } + + const state = { + name, + kind, + type, + source, + version, + mappings, + state: props, + module: getModuleId() ?? '__global', + testContext: getTestContext(), + scopes: globalFunctions.getScopes(), + } + + const proxy = createEntityProxy(this, state, mappings) + getState().registered.set(getElementKey(state), proxy) + + return proxy + } + } as any + + if (kind === 'provider') { + Object.defineProperty(c, Symbol.for('contextType'), { + value: type, + enumerable: true, + }) + } + + return c +} + +const synapseOutput = Symbol.for('synapseClassOutput') + +export function createSynapseClass( + type: string, + kind: 'resource' | 'data-source' | 'provider' = 'resource' +): { new (config: T): U } { + const tfType = kind === 'provider' ? 'synapse' : 'synapse_resource' + + const cls = class extends createTerraformClass(tfType, kind) { + static [terraformClass] = true + + public constructor(config: T) { + if (kind === 'provider') { + super(config) + + return + } + + super({ type, input: config }) + + const _this = this + + return new Proxy(_this, { + get: (target, prop, recv) => { + if (prop === synapseOutput) { + return _this.output + } + + if (Reflect.has(target, prop) || typeof prop === 'symbol') { + return Reflect.get(target, prop, recv) + } + + return _this.output[prop] + }, + }) + } + } + + return Object.defineProperty(cls, 'name', { + value: type, + writable: false, + configurable: true, + enumerable: false + }) as any +} + +const functions = { + jsonencode: (obj: any) => '' as string, + encoderesource: (obj: any) => '' as string, + serialize: (obj: any) => ({}) as any, + jsondecode: (str: string) => ({}) as any, + dirname: (path: string) => '' as string, + trimprefix: (target: string, prefix: string) => '' as string, + replace: (str: string, substring: string | RegExp, replacement: string) => '' as string, + basename: (str: string) => '' as string, + abspath: (str: string) => '' as string, + substr: (str: string, offset: number, length: number) => '' as string, + element: (arr: T[], index: number) => ({}) as T, + tolist: (arr: T[]) => ({}) as T[], + generateidentifier: (targetId: string, attribute: string, maxLength: number, sep?: string) => '' as string, +} + +export const Fn = createFunctions(k => (...args: any[]) => createCallExpression(k, args, { + ['__serializer']: getSerializer({ + moduleId: getModuleId() ?? '__global', + testContext: getTestContext(), + dataTable: {}, + }), +})) + +// export declare function registerBeforeSynthHook(callback: SynthHook): void + +function createFunctions(factory: (name: T) => (typeof functions)[T]): typeof functions { + return Object.fromEntries( + Object.keys(functions).map(k => [k, factory(k as any)]) + ) as typeof functions +} + +export interface State { + registered: Map + backends: Map + moved: { from: string, to: string }[] + names: Set + serialized: Map + tables: Map + serializers: Map> + secrets: Map +} + +const terraformClass = Symbol.for('terraformClass') + +let globalFunctions: { + getState: () => State, + getScopedId: (peek?: boolean) => string | undefined, + getModuleId: () => string | undefined + getProviders: () => Record + exportSymbols?: (getSymbolId: (sym: Symbol) => number) => void + getScopes: () => ExecutionScope[] +} + +function assertInit() { + if (typeof globalFunctions === 'undefined') { + throw new Error(`Cannot be called outside of the compiler`) + } +} + +function getState() { + assertInit() + + return globalFunctions.getState() +} + +function getScopedId(peek?: boolean) { + assertInit() + + return globalFunctions.getScopedId(peek) +} + +function getProviders() { + assertInit() + + return globalFunctions.getProviders() +} + +function getModuleId() { + assertInit() + + return globalFunctions.getModuleId() +} + +const defaultSerializerName = 'default' +function getSerializer(ctx: Context) { + const serializers = getState().serializers + if (serializers.has(defaultSerializerName)) { + return serializers.get(defaultSerializerName)!.withContext(ctx) + } + + const serializer = createSerializer(getState().serialized, getState().tables) + serializers.set(defaultSerializerName, serializer) + + return serializer.withContext(ctx) +} + +function peekName(type: string, kind: TerraformElement['kind'] | 'local', prefix?: string, suffix?: string) { + let count = 0 + const resolvedPrefix = prefix ?? `${kind === 'provider' ? '' : `${kind}-`}${type}` + const cleanedPrefix = resolvedPrefix.replace(/\$/g, 'S-') // XXX + const getName = () => `${cleanedPrefix || 'default'}${suffix ? `--${suffix}` : ''}${count === 0 ? '' : `-${count}`}` + while (getState().names.has(`${type}.${getName()}`)) count++ + + return getName() + + // if (kind === 'provider' && isDefaultProviderName(finalName, type)) { + // return finalName + // } + + // return 'r-' + require('crypto').createHash('sha256').update(finalName).digest('hex') +} + +function generateName(type: string, kind: TerraformElement['kind'] | 'local', prefix?: string, suffix?: string) { + const finalName = peekName(type, kind, prefix, suffix) + getState().names.add(`${type}.${finalName}`) + + return finalName +} + +export function init( + getState: () => State, + getScopedId: (peek?: boolean) => string | undefined, + getModuleId: () => string | undefined, + getProviders: () => Record, + getScopes: () => ExecutionScope[], + exportSymbols?: (getSymbolId: (sym: Symbol) => number) => void, +) { + globalFunctions = { + getState, + getScopedId, + getProviders, + getModuleId, + getScopes, + exportSymbols, + } + + function registerBackend(type: string, config: any) { + getState().backends.set(type, config) + } + + function getResources(moduleName: string, includeTests = false) { + const r: any[] = [] + for (const [k, v] of getState().registered) { + const element = v[internalState] as InternalState + if (element.testContext?.id !== undefined && !includeTests) { + continue + } + if (element.module === moduleName) { + r.push(v) + } + } + + return r + } + + const beforeSynthHooks: SynthHook[] = [] + function registerBeforeSynthHook(...callbacks: SynthHook[]) { + beforeSynthHooks.push(...callbacks) + } + + function registerSecret(envVar: string, type: string) { + getState().secrets.set(envVar, type) + } + + const sourceMap: TerraformSourceMap = { + symbols: [], + resources: {} + } + + const symbolIds = new Map() + function getSymbolId(symbol: Symbol) { + const key = `${symbol.fileName}:${symbol.line}:${symbol.column}` + if (!symbolIds.has(key)) { + symbolIds.set(key, symbolIds.size) + sourceMap.symbols.push(symbol) + } + + return symbolIds.get(key)! + } + + function addSymbols(resourceName: string, scopes: ExecutionScope[]) { + const relevantScopes = scopes.filter(s => !!s.callSite) + const mapped = relevantScopes.map(s => ({ + isNewExpression: s.isNewExpression, + callSite: getSymbolId(s.callSite!), + assignment: s.assignment ? getSymbolId(s.assignment) : undefined, + namespace: s.namespace?.map(getSymbolId), + })) + + if (mapped.length > 0) { + sourceMap.resources[resourceName] = { scopes: mapped } + } + } + + const sourceMapper: SourceMapper = { + addSymbols, + getSourceMap: () => sourceMap, + } + + return { + getResources, + registerSecret, + registerBackend, + registerBeforeSynthHook, + emitTerraformJson: () => { + if (globalFunctions.exportSymbols) { + globalFunctions.exportSymbols(getSymbolId) + } + + return emitTerraformJson(getState(), sourceMapper, { before: beforeSynthHooks }) + } + } +} diff --git a/src/runtime/modules/test.ts b/src/runtime/modules/test.ts new file mode 100644 index 0000000..1a082bf --- /dev/null +++ b/src/runtime/modules/test.ts @@ -0,0 +1,271 @@ +//# moduleId = synapse:test +//# transform = persist + +import * as assert from 'node:assert' +import { using, defer, getCurrentId, contextType, maybeGetContext, importArtifact } from 'synapse:core' +import { createSynapseClass } from 'synapse:terraform' +import { Export } from 'synapse:lib' + +// Starting at one to guard against erroneous falsy checks on the id +let idCounter = 1 + +export function test(name: string, fn: () => Promise | void): void { + const suite = maybeGetContext(TestSuite) + if (suite) { + suite.addTest(new Test(idCounter++, name, fn)) + } else { + addDeferredTestItem({ type: 'test', name, fn }) + } +} + +export function it(name: string, fn: () => Promise | void): void { + return test(name, fn) +} + +export function describe(name: string, fn: () => void): void { + const suite = maybeGetContext(TestSuite) + if (suite) { + const id = getCurrentId() + const child = new TestSuite(idCounter++, name) + visitTestSuite(id, child, fn) + } else { + addDeferredTestItem({ type: 'suite', name, fn }) + } +} + +interface TestProps { + readonly id: number + readonly name: string + readonly handler: string // pointer +} + +interface TestOutput { + readonly id: number + readonly name: string + readonly handler: string // pointer +} + +class TestResource extends createSynapseClass('Test') {} +class TestSuiteResource extends createSynapseClass('TestSuite') {} + +export class Test { + private readonly resource: TestResource + + public constructor(public readonly id: number, public readonly name: string, fn: () => Promise | void) { + const handler = new Export({ fn }) + this.resource = new TestResource({ + id, + name, + handler: handler.destination, + }) + } + + public async run() { + const { fn } = await importArtifact(this.resource.handler) + await fn() + } +} + +export class TestSuite { + static readonly [contextType] = 'test-suite' + public readonly tests: Test[] = [] + + public constructor( + public readonly id: number, // `id` is needed when serializing + public readonly name: string, + ) {} + + public addTest(test: Test) { + this.tests.push(test) + } + + public async run() { + const results: { name: string; error?: Error }[] = [] + for (const test of this.tests) { + const { name } = test + + try { + await test.run() + results.push({ name }) + } catch (e) { + results.push({ name, error: e as Error }) + } + } + + return results + } +} + +function visitTestSuite(id: string, suite: TestSuite, fn: () => void) { + const handler = using(suite, () => { + fn() + + return new Export({ suite }, { + id: id + '--test-suite', + testSuiteId: suite.id, + }) + }) + + new TestSuiteResource({ + id: suite.id, + name: suite.name, + handler: handler.destination, + }) +} + +declare function __getCallerModuleId(): string | undefined + +interface DeferredTest { + type: 'test' + name: string + fn: () => Promise | void +} + +interface DeferredSuite { + type: 'suite' + name: string + fn: () => void +} + +type DeferredTestItem = DeferredTest | DeferredSuite + +const deferredTestsItems = new Map() +function addDeferredTestItem(item: DeferredTestItem) { + const caller = __getCallerModuleId() + if (!caller) { + throw new Error(`Missing caller module id`) + } + + if (deferredTestsItems.has(caller)) { + deferredTestsItems.get(caller)!.push(item) + return + } + + const arr: DeferredTestItem[] = [] + arr.push(item) + deferredTestsItems.set(caller, arr) + + defer(() => { + const suite = new TestSuite(idCounter++, caller) + + using(suite, () => { + for (const item of arr) { + if (item.type === 'test') { + suite.addTest(new Test(idCounter++, item.name, item.fn)) + } else { + const id = getCurrentId() + const child = new TestSuite(idCounter++, item.name) + visitTestSuite(id, child, item.fn) + } + } + + const handler = new Export({ suite }, { + id: caller + '--test-suite', + testSuiteId: suite.id, + }) + + new TestSuiteResource({ + id: suite.id, + name: suite.name, + handler: handler.destination, + }) + }) + }) +} + +// First operand -> actual +// Second operand (optional) -> expected +// +// We're just wrapping `node:assert` currently + +function createAssertionError( + actual?: unknown, + expected?: unknown, + operator?: 'strictEqual' | 'deepStrictEqual' | '==', + message?: string +) { + + // We just want the error message, not the stack trace + const err = new assert.AssertionError({ message, actual, expected, operator }) + + return new Error(err.message) +} + +function assertOk(value: unknown, message?: string) { + if (!value) { + throw createAssertionError(value, true, '==', message) + } +} + +function assertStrictEqual(actual: unknown, expected: unknown) { + if (!Object.is(actual, expected)) { + throw createAssertionError(actual, expected, 'strictEqual') + } +} + +function deepArrayEqual(actual: unknown[], expected: unknown[], message?: string) { + if (actual.length !== expected.length) { + throw createAssertionError(actual, expected, 'deepStrictEqual', message) + } + + for (let i = 0; i < actual.length; i++) { + assertDeepStrictEqual(actual[i], expected[i], message) + } +} + +// Not the best impl. +// This was quickly thrown together +// TODO: check proto, check descriptors +function assertDeepStrictEqual(actual: unknown, expected: unknown, message?: string) { + if (typeof actual !== typeof expected) { + throw createAssertionError(typeof actual, typeof expected, 'deepStrictEqual', message) + } + + if (Object.is(actual, expected)) { + return + } + + if (actual === null || expected === null) { + throw createAssertionError(actual, expected, 'deepStrictEqual', message) + } + + if (typeof actual !== 'object') { + throw createAssertionError(actual, expected, 'deepStrictEqual', message) + } + + if (Array.isArray(actual) || Array.isArray(expected)) { + if (!Array.isArray(actual) || !Array.isArray(expected)) { + throw createAssertionError(actual, expected, 'deepStrictEqual', message) + } + + return deepArrayEqual(actual, expected, message) + } + + const actualKeys = Object.getOwnPropertyNames(actual).sort() + const expectedKeys = Object.getOwnPropertyNames(expected).sort() + deepArrayEqual(actualKeys, expectedKeys, message) + + for (const k of actualKeys) { + assertDeepStrictEqual((actual as any)[k], (expected as any)[k], message) + } +} + +export function expect(actual: unknown, message?: string): asserts actual { + assertOk(actual, message) +} + +export function expectEqual(actual: unknown, expected: T, message?: string): asserts actual is T { + assertDeepStrictEqual(actual, expected, message) +} + +export function expectReferenceEqual(actual: unknown, expected: T): asserts actual is T { + assertStrictEqual(actual, expected) +} + +// TODO: the types don't make sense here +// export function expectReject(actual: Promise, expected: T, message?: string): asserts actual is Promise { +// let didReject = false +// const res = waitForPromise(actual.catch(e => (didReject = true, e))) +// expect(didReject, message ?? 'Promise did not reject') +// expectEqual(res, expected, message) +// } diff --git a/src/runtime/modules/validation.ts b/src/runtime/modules/validation.ts new file mode 100644 index 0000000..4b33d0c --- /dev/null +++ b/src/runtime/modules/validation.ts @@ -0,0 +1,302 @@ +//@internal +//# moduleId = synapse:validation +// TODO: turn this into a 'reify' lib + +type PrimitiveType = 'null' | 'boolean' | 'object' | 'array' | 'number' | 'string' // | 'integer' +type InstanceType = PrimitiveType | 'integer' | (PrimitiveType | 'integer' )[] + +interface NumberSchema { + readonly multipleOf?: number + readonly maximum?: number + readonly minimum?: number + readonly exclusiveMaximum?: number + readonly exclusiveMinimum?: number +} + +interface StringSchema { + readonly pattern?: string // RegExp + readonly maxLength?: number + readonly minLength?: number +} + +interface ArraySchema { + // readonly type?: InstanceType + readonly maxItems?: number + readonly minItems?: number + readonly uniqueItems?: boolean + + // readonly maxContains?: number + // readonly minContains?: number +} + +interface ObjectSchema { + readonly maxProperties?: number + readonly minProperties?: number + readonly required?: string[] + readonly dependentRequired?: Record + + // readonly maxContains?: number + // readonly minContains?: number +} + +// Basic impl. of JSON schema + +interface SchemaBase { + readonly type?: PrimitiveType | PrimitiveType[] + readonly enum?: (null | boolean | string | number)[] + readonly const?: null | boolean | string | number + readonly anyOf?: Schema[] +} + +interface ObjectSchema extends SchemaBase { + readonly type: 'object' + readonly properties?: Record + readonly additionalProperties?: Schema + readonly required?: string[] +} + +interface ArraySchema extends SchemaBase { + readonly type: 'array' + readonly items?: Schema | false + readonly prefixItems?: Schema[] +} + +interface StringSchema extends SchemaBase { + readonly type: 'string' +} + +interface NumberSchema extends SchemaBase { + readonly type: 'number' +} + +interface BooleanSchema extends SchemaBase { + readonly type: 'boolean' +} + +interface NullSchema extends SchemaBase { + readonly type: 'null' +} + +export interface TypedObjectSchema extends ObjectSchema { + readonly __type: T +} + +export interface TypedArraySchema extends ArraySchema { + readonly __type: T +} + +export interface TypedStringSchema extends StringSchema { + readonly __type: T +} + +export interface TypedNumberSchema extends NumberSchema { + readonly __type: T +} + +export type Schema = ObjectSchema | ArraySchema | StringSchema | NumberSchema | BooleanSchema | NullSchema + +export type FromSchema = T extends TypedArraySchema ? U + : T extends TypedObjectSchema ? U + : T extends TypedStringSchema ? U + : T extends TypedNumberSchema ? U : never + +export function validate(val: unknown, schema: TypedArraySchema): asserts val is T +export function validate(val: unknown, schema: TypedObjectSchema): asserts val is T +export function validate(val: unknown, schema: TypedStringSchema): asserts val is T +export function validate(val: unknown, schema: TypedNumberSchema): asserts val is T + +export function validate(val: unknown, schema: Schema) { + checkSchema(val, schema)?.throw() +} + +export function checkSchema(val: unknown, schema: Schema): ValidationError | void { + if (schema.anyOf) { + const errors: ValidationError[] = [] + for (const subschema of schema.anyOf) { + const expanded = { ...schema, anyOf: undefined, ...subschema } + const error = checkSchema(val, expanded) + if (!error) { + return + } + + errors.push(error) + } + + return new ValidationError('Failed to match any subschemas', val, schema, errors) + } + + if (Array.isArray(schema.type)) { + const errors: ValidationError[] = [] + for (const type of schema.type) { + const error = checkSchema(val, { ...schema, type }) + if (!error) { + return + } + + errors.push(error) + } + + return new ValidationError(`Failed to match any types: ${schema.type}`, val, schema, errors) + } + + if (schema.enum) { + const match = schema.enum.indexOf(val as any) + if (!match) { + return new ValidationError(`Value must be on of: ${schema.enum}`, val, schema) + } + + return + } else if (schema.const && schema.const !== val) { + return new ValidationError(`Value must be equal to ${schema.const}`, val, schema) + } + + switch (schema.type) { + case 'number': + case 'string': + case 'boolean': + if (typeof val !== schema.type) { + return new ValidationError(`Expected value with type: ${schema.type}, got ${typeof val}`, val, schema) + } + + break + case 'null': + if (val !== null) { + return new ValidationError(`Expected null`, val, schema) + } + + break + case 'array': { + if (!Array.isArray(val)) { + return new ValidationError(`Expected an array, got ${typeof val}`, val, schema) + } + + let i = 0 + const errors: ValidationError[] = [] + const prefixItems = schema.prefixItems + if (prefixItems) { + for (; i < prefixItems.length; i++) { + const error = checkSchema(val[i], prefixItems[i]) + if (error) { + errors.push(error) + } + } + + if ((!schema.items && prefixItems.length > val.length) || val.length < prefixItems.length) { + return new ValidationError( + `Incorrect number of items in array: got ${val.length}, expected ${prefixItems.length}`, + val, + schema, + errors + ) + } + } + + if (schema.items) { + for (; i < val.length; i++) { + const error = checkSchema(val[i], schema.items) + if (error) { + errors.push(error) + } + } + } + + if (errors.length > 0) { + return new ValidationError( + `Failed to validate array`, + val, + schema, + errors + ) + } + + break + } + case 'object': { + if (typeof val !== 'object' || val === null) { + return new ValidationError( + `Expected an object, got ${val === null ? 'null' : typeof val}`, + val, + schema, + ) + } + + const errors: ValidationError[] = [] + const matched = new Set() + if (schema.properties) { + for (const [k, v] of Object.entries(schema.properties)) { + matched.add(k) + + const isRequired = !!schema.required?.includes(k) + const subval = (val as any)[k] + if (subval === undefined) { + if (isRequired) { + errors.push(new ValidationError(`${k}: missing value`, val, v)) + } + + continue + } + + const error = checkSchema(subval, v) + if (error) { + errors.push(new ValidationError(`${k}: ${error.message}`, subval, v, Array.from(error))) + } + } + } + + if (schema.additionalProperties) { + for (const [k, v] of Object.entries(val)) { + if (matched.has(k)) { + continue + } + + const error = checkSchema(v, schema.additionalProperties) + if (error) { + errors.push(new ValidationError(`${k}: ${error.message}`, v, schema.additionalProperties, Array.from(error))) + } + } + } + + if (errors.length > 0) { + return new ValidationError('Failed to validate object', val, schema, errors) + } + + break + } + } +} + +class AggregateError extends Error implements Iterable { + constructor(message: string, private readonly errors: T[]) { + super(message) + } + + public [Symbol.iterator]() { + return this.errors[Symbol.iterator]() + } +} + +class ValidationError extends AggregateError { + public readonly name = 'ValidationError' + + constructor(message: string, value: unknown, schema: Schema, reasons: ValidationError[] = []) { + super(message, reasons) + } + + // Only used to format the error message + public throw(): never { + const message = format(this) + const error = new Error(message) + error.name = this.name + + throw error + } +} + +function format(error: ValidationError, depth = 0): string { + const message = [ + error.message, + ...Array.from(error).map(e => format(e, depth + 1)) + ] + + return message.map(s => `${' '.repeat(depth)}${s}`).join('\n') +} \ No newline at end of file diff --git a/src/runtime/modules/ws.ts b/src/runtime/modules/ws.ts new file mode 100644 index 0000000..41b45fc --- /dev/null +++ b/src/runtime/modules/ws.ts @@ -0,0 +1,939 @@ +//# moduleId = synapse:ws +//# transform = persist + +import * as http from 'node:http' +import * as https from 'node:https' +import * as crypto from 'node:crypto' +import * as net from 'node:net' +import { EventEmitter } from 'node:events' +import { Duplex } from 'node:stream' +import * as tls from 'node:tls' +import { HttpError, HttpRequest, HttpResponse } from 'synapse:http' + +const WebSocketGUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' +function getWsKey(key: string) { + return crypto.createHash('sha1').update(key + WebSocketGUID).digest().toString('base64') +} + +const slice = (buf: Buffer, start: number, end?: number) => Uint8Array.prototype.slice.call(buf, start, end) + +interface WebSocketServer { + close: () => Promise + onConnect: (listener: (ws: WebSocket) => void) => { dispose: () => void } +} + +interface WebSocketServerOptions { + port?: number + address?: string + secureContext?: tls.SecureContextOptions + beforeUpgrade?: (request: HttpRequest) => Promise | HttpResponse | void + httpRequestHandler?: (request: HttpRequest, body: any) => Promise | HttpResponse +} + +export async function upgradeToWebsocket(req: http.IncomingMessage, socket: Duplex, isSecureContext = true): Promise { + const key = req.headers['sec-websocket-key'] + const protocol = req.headers['sec-websocket-protocol'] + if (!key) { + console.log('No key') + socket.destroy() + return + } + + const headers = new Headers({ + Upgrade: 'websocket', + Connection: 'Upgrade', + 'Sec-WebSocket-Accept': getWsKey(key), + // 'Sec-WebSocket-Protocol': protocol ? protocol.split(',')[0] : '', + }) + + function getBaseUrl() { + const scheme = isSecureContext ? 'wss' : 'ws' + + return scheme + '://' + req.headers['host']! + } + + // TODO: clean this up + const url = new URL(req.url!, getBaseUrl()) + + const resp = [ + 'HTTP/1.1 101 Switching Protocols', + ...Array.from(headers.entries()).map(([k, v]) => `${k}: ${v}`), + '', + ].join('\r\n') + + + return new Promise((resolve, reject) => { + socket.write(resp + '\r\n', err => { + if (!err) { + resolve(upgradeSocket(socket, url)) + } else { + reject(err) + } + }) + }) +} + +/** @internal */ +export async function createWebsocketServer(opt: WebSocketServerOptions = {}): Promise { + const server = opt?.secureContext + ? https.createServer(opt.secureContext) + : http.createServer() + + const emitter = new EventEmitter() + const connectEvent: Event<[ws: WebSocket]> = createEvent(emitter, 'connect') + + server.on('request', async (req, res) => { + if (!opt.httpRequestHandler) { + return res.end('') + } + + const body = await new Promise((resolve, reject) => { + const buffer: Buffer[] = [] + req.once('error', reject) + req.on('data', chunk => buffer.push(chunk)) + req.on('end', () => resolve(buffer.join(''))) + }) + + try { + const resp = await opt.httpRequestHandler({ + // body: body as any, + path: req.url!, + method: req.method!, + pathParameters: {}, + headers: new Headers(Object.entries(req.headers).filter(([_, v]) => typeof v === 'string') as any), + }, body) + + if (resp.body) { + throw new Error('Not implemented') + } + + res.writeHead(resp.statusCode ?? 200, resp.headers) + res.end('') + } catch (e) { + if (e instanceof HttpError) { + const body = JSON.stringify({ message: e.message }) + res.writeHead(e.fields.statusCode, { + 'content-type': 'application/json', + 'content-length': body.length, + }) + res.end(body) + + return + } + + res.writeHead(500) + res.end('') + + throw e + } + }) + + const sockets = new Set() + server.on('upgrade', async (req, socket, head) => { + const beforeUpgradeResp = await opt.beforeUpgrade?.({ + // body: undefined, + path: req.url!, + method: req.method!, + pathParameters: {}, + headers: new Headers(Object.entries(req.headers).filter(([_, v]) => typeof v === 'string') as any), + }) + + if (beforeUpgradeResp) { + const code = beforeUpgradeResp.statusCode ?? 400 + const msg = http.STATUS_CODES[code] ?? 'Unknown' + const headers = new Headers(beforeUpgradeResp.headers ?? {}) + + const resp = [ + `HTTP/1.1 ${code} ${msg}`, + ...Array.from(headers.entries()).map(([k, v]) => `${k}: ${v}`), + '', + ].join('\r\n') + + return socket.end(resp + '\r\n') + } + + const ws = await upgradeToWebsocket(req, socket, !!opt.secureContext) + if (!ws) { + return + } + + const l = ws.onClose(() => (l.dispose(), sockets.delete(ws))) + sockets.add(ws) + connectEvent.fire(ws) + }) + + async function close() { + await Promise.all(Array.from(sockets).map(ws => ws.close())) + + return new Promise((resolve, reject) => + server.close(err => err ? reject(err) : resolve()) + ) + } + + function listen(address?: string) { + return new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(port, address, () => { + server.removeListener('error', reject) + resolve() + }) + }) + } + + const port = opt.port ?? (opt.secureContext ? 443 : 80) + if (opt.address === '*') { + await Promise.all([listen('::'), listen('0.0.0.0')]) + } else { + await listen(opt.address) + } + + return { + close, + onConnect: connectEvent.on, + } +} + +// FIXME: conform with WHATWG WebSocket +export interface WebSocket { + readonly url: URL + ping: (message?: string) => Promise + send: (data: string | Uint8Array) => Promise + close: (message?: string) => Promise + onClose: (listener: (code?: StatusCode, reason?: string) => void) => { dispose: () => void } + onMessage: (listener: (message: string | Buffer) => void) => { dispose: () => void } + onError: (listener: (error: unknown) => void) => { dispose: () => void } +} + +interface Event { + fire(...args: T): void + on(listener: (...args: T) => void): { dispose: () => void } + once(listener: (...args: T) => void): { dispose: () => void } +} + +function createEvent(emitter: EventEmitter, type: U): Event { + return { + fire: (...args) => emitter.emit(type, ...args), + on: listener => { + emitter.on(type, listener as any) + + return { dispose: () => void emitter.removeListener(type, listener as any) } + }, + once: listener => { + emitter.once(type, listener as any) + + return { dispose: () => void emitter.removeListener(type, listener as any) } + }, + } +} + +function upgradeSocket(socket: Duplex, url: URL, mode: 'client' | 'server' = 'server'): WebSocket { + const emitter = new EventEmitter() + const pongEvent: Event<[message: string]> = createEvent(emitter, 'pong') + const errorEvent: Event<[error: unknown]> = createEvent(emitter, 'error') + const closeEvent: Event<[code?: StatusCode, reason?: string]> = createEvent(emitter, 'close') + const messageEvent: Event<[message: string | Buffer]> = createEvent(emitter, 'message') + + function ping(msg: string = '') { + const now = Date.now() + + return new Promise((resolve, reject) => { + const l = pongEvent.once(() => resolve(Date.now() - now)) + socket.write( + createFrame(Opcode.Ping, Buffer.from(msg)), + err => err ? (l.dispose(), reject(err)) : void 0, + ) + }) + } + + function randomInt32() { + return new Promise((resolve, reject) => { + crypto.randomInt(0, 2 ** 32, (err, value) => { + if (err) { + reject(err) + } else { + resolve(value) + } + }) + }) + } + + async function send(msg: string | Uint8Array) { + const maskingKey = mode === 'client' ? await randomInt32() : undefined + + return new Promise((resolve, reject) => { + socket.write( + createFrame(Opcode.Text, Buffer.from(msg), maskingKey), + err => err ? reject(err) : resolve() + ) + }) + } + + async function close(reason?: string) { + state = SocketState.Closing + + try { + const closePromise = new Promise((resolve, reject) => { + socket.once('end', () => resolve()) + // socket.once('error', reject) + }) + + await sendClose(StatusCode.Success, reason) + await closePromise + } finally { + state = SocketState.Closed + } + } + + async function sendClose(code?: StatusCode, reason?: string | Uint8Array) { + return new Promise((resolve, reject) => { + socket.write( + createCloseFrame(code, reason), + err => err ? reject(err) : resolve() + ) + }) + } + + let state = SocketState.Ready + let lastFrame: Frame | undefined + let messageBuffer: Buffer[] = [] + + async function handleFrame(frame: Frame) { + switch (frame.op) { + case Opcode.Pong: + return pongEvent.fire(frame.payload.toString('utf-8')) + case Opcode.Ping: + return socket.write(createFrame(Opcode.Pong, frame.payload)) + case Opcode.Close: { + let code: StatusCode | undefined + let reason: Buffer | undefined + + if (frame.payload.length >= 2) { + code = frame.payload.readUint16BE() + reason = Buffer.from(slice(frame.payload, 2)) + } + + closeEvent.fire(code, reason?.toString('utf-8')) + + if (state !== SocketState.Closing && state !== SocketState.Closed) { + // console.log('closing', code, reason) + state = SocketState.Closing + + try { + await sendClose(code, reason) + } finally { + socket.end(() => state = SocketState.Closed) + } + } + + return + } + case Opcode.Text: + case Opcode.Binary: { + if (state !== SocketState.Ready) { + throw new Error(`Unexpected socket state: ${state}`) + } + + if (frame.fin) { + const res = frame.op === Opcode.Text ? frame.payload.toString('utf-8') : frame.payload + messageEvent.fire(res) + } else { + messageBuffer.push(frame.payload) + state = frame.op === Opcode.Text ? SocketState.ReadingText : SocketState.ReadingBinary + } + + return + } + case Opcode.Continue: { + if (state !== SocketState.ReadingText && state !== SocketState.ReadingBinary) { + throw new Error(`Unexpected continuation state: ${state}`) + } + + if (frame.fin) { + const res = state === SocketState.ReadingText + ? messageBuffer.join('') + : Buffer.concat(messageBuffer) + messageEvent.fire(res) + messageBuffer = [] + state = SocketState.Ready + } else { + messageBuffer.push(frame.payload) + } + + return + } + default: + return sendClose(StatusCode.ProtocolError, `unknown opcode: ${frame.op}`) + } + } + + socket.on('error', errorEvent.fire) + socket.on('end', () => { + if (state === SocketState.Closing) { + emitter.removeAllListeners() + return + } + + if (state !== SocketState.Closed) { + closeEvent.fire(StatusCode.UnexpectedCloseState) + state = SocketState.Closed + } + + emitter.removeAllListeners() + }) + socket.on('data', async (buf: Buffer) => { + let frame: Frame + if (lastFrame) { + const [_, remainingLength] = readFrameIntoPayload(buf, lastFrame.remainingLength!, 0, lastFrame.maskingKey, lastFrame.payload, lastFrame.payload.length - lastFrame.remainingLength!) + + lastFrame.remainingLength = remainingLength + if (lastFrame.remainingLength) { + return + } + + frame = lastFrame + lastFrame = undefined + } else { + frame = readFrame(buf) + } + + if (frame.remainingLength) { + lastFrame = frame + return + } + + try { + await handleFrame(frame) + } catch (e) { + errorEvent.fire(e) + } + }) + + return { + url, + ping, + send, + close, + onClose: closeEvent.on, + onError: errorEvent.on, + onMessage: messageEvent.on, + } +} + +/** @internal */ +export async function createWebsocket(url: string | URL, authorization?: string): Promise { + const parsedUrl = new URL(url) + if (parsedUrl.protocol !== 'ws:' && parsedUrl.protocol !== 'wss:') { + throw new Error(`Invalid protocol "${parsedUrl.protocol}". Must be one of: ws:, wss:`) + } + + const port = Number(parsedUrl.port || (parsedUrl.protocol === 'wss:' ? 443 : 80)) + const socket = await new Promise((resolve, reject) => { + let s: Duplex + + function complete(err?: Error) { + if (err) { + reject(err) + } else { + resolve(s) + } + + s.removeListener('error', complete) + } + + if (parsedUrl.protocol === 'ws:') { + s = net.connect({ + port, + host: parsedUrl.hostname, + }, complete) + } else { + s = tls.connect({ + port, + host: parsedUrl.hostname, + servername: parsedUrl.hostname, + }, complete) + } + + s.on('error', complete) + }) + + const nonce = crypto.randomBytes(16).toString('base64') + const headers = new Headers({ + Host: parsedUrl.hostname, + Upgrade: 'websocket', + Connection: 'upgrade', + 'Sec-WebSocket-Key': nonce, + 'Sec-WebSocket-Version': '13', + // 'Sec-WebSocket-Protocol': protocol ? protocol.split(',')[0] : '', + }) + + if (authorization) { + headers.set('Authorization', authorization) + } + + const req = [ + `GET ${parsedUrl.pathname} HTTP/1.1`, + ...Array.from(headers.entries()).map(([k, v]) => `${k}: ${v}`), + '', + ].join('\r\n') + + return new Promise(async (resolve, reject) => { + function complete(err?: Error) { + if (err) { + reject(err) + } else { + resolve(upgradeSocket(socket, parsedUrl, 'client')) + } + + socket.removeListener('data', onData) + socket.removeListener('error', complete) + } + + let buffer = '' + function onData(chunk: Buffer) { + buffer += chunk + if (buffer.endsWith('\r\n\r\n')) { + const lines = buffer.split('\r\n').slice(0, -1) + if (lines.length === 0) { + return complete(new Error(`Missing or malformed HTTP response`)) + } + + const [httpVersion, statusCode, message] = lines[0].split(' ') + if (!httpVersion || !statusCode) { + return complete(new Error(`Malformed status line: ${lines[0]}`)) + } + + // TODO: handle redirect + if (statusCode !== '101') { + return complete(new Error(`Server rejected upgrade request: ${statusCode} ${message}`)) + } + + const headers = Object.fromEntries( + lines.slice(1) + .map(l => l.split(': ', 2)) + .filter(([_, v]) => v !== undefined) + .map(([k, v]) => [k.toLowerCase(), v]) + ) as Record + + if (headers['upgrade']?.toLowerCase() !== 'websocket') { + return complete(new Error(`Invalid upgrade header: ${headers['upgrade']}`)) + } + + if (headers['connection']?.toLowerCase() !== 'upgrade') { + return complete(new Error(`Invalid connection header: ${headers['connection']}`)) + } + + // TODO: subprotocols + if (headers['sec-websocket-protocol']) { + return complete(new Error(`Invalid subprotocol: ${headers['sec-websocket-protocol']}`)) + } + + complete() + } + } + + socket.on('error', complete) + socket.write(req + '\r\n', err => { + if (!err) { + socket.on('data', onData) + } + }) + + setTimeout(() => { + if (socket.listeners('error').includes(complete)) { + socket.emit('error', new Error('Timed out waiting for connection')) + } + }, 10000).unref() + }) +} + +// https://stackoverflow.com/a/23329386 +function byteLength(str: string | Uint8Array) { + if (typeof str !== 'string') { + return str.length + } + + let len = str.length + for (let i = str.length - 1; i >= 0; i--) { + const code = str.charCodeAt(i) + if (code >= 0xDC00 && code <= 0xDFFF) i-- + + if (code > 0x7F && code <= 0x7FF) len++ + else if (code > 0x7FF && code <= 0xFFFF) len += 2 + } + + return len +} + + +enum SocketState { + Ready = 0, + ReadingText = 1, + ReadingBinary = 2, + Closing = 3, + Closed = 4, + Truncated = 10, +} + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-------+-+-------------+-------------------------------+ +// |F|R|R|R| opcode|M| Payload len | Extended payload length | +// |I|S|S|S| (4) |A| (7) | (16/64) | +// |N|V|V|V| |S| | (if payload len==126/127) | +// | |1|2|3| |K| | | +// +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - + +// | Extended payload length continued, if payload len == 127 | +// + - - - - - - - - - - - - - - - +-------------------------------+ +// | |Masking-key, if MASK set to 1 | +// +-------------------------------+-------------------------------+ +// | Masking-key (continued) | Payload Data | +// +-------------------------------- - - - - - - - - - - - - - - - + +// : Payload Data continued ... : +// + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// | Payload Data continued ... | +// +---------------------------------------------------------------+ + +const maxFrameSize = 65536 + +function *chunk(buf: Buffer, amount: number) { + let pos = 0 + while (true) { + const start = pos === 0 + const c = Uint8Array.prototype.slice.call(buf, pos, Math.min(pos + amount, buf.length)) + pos += c.length + + const fin = pos >= buf.length + yield [c, start, fin] + if (fin) break + } +} + +function createDataFrames(payload: Buffer, maskingKey?: number) { + +} + +function createCloseFrame(code?: StatusCode, reason?: string | Uint8Array) { + if (code === undefined) { + return createFrame(Opcode.Close) + } + + const size = (reason ? byteLength(reason) : 0) + 2 + const payload = Buffer.allocUnsafe(size) + payload.writeUint16BE(code) + if (typeof reason === 'string') { + payload.write(reason, 2) + } else if (reason instanceof Uint8Array) { + payload.set(reason, 2) + } + + return createFrame(Opcode.Close, payload) +} + +function createFrame(op: Opcode, payload?: Buffer, maskingKey?: number, fin = true) { + if (payload === undefined) { + return Buffer.of(((fin ? 1 : 0) << 7) | op) + } + + let pos = 0 + + const mask = maskingKey !== undefined + const p = payload.length <= 125 ? payload.length : payload.length <= 65536 ? 126 : 127 + const size = 2 + payload.length + (mask ? 4 : 0) + (p === 126 ? 2 : p === 127 ? 8 : 0) + + const frame = Buffer.allocUnsafe(size) + pos = frame.writeUInt8(((fin ? 1 : 0) << 7) | op, pos) + pos = frame.writeUint8((mask ? 1 : 0) << 7 | p, pos) + if (p === 126) { + pos = frame.writeUint16BE(payload.length, pos) + } + if (p === 127) { + pos = frame.writeBigUint64BE(BigInt(payload.length), pos) + } + if (mask) { + pos = frame.writeUint32BE(maskingKey, pos) + let i: number + for (i = 0; i < payload.length - 4; i += 4) { + pos = frame.writeInt32BE(payload.readInt32BE(i) ^ maskingKey, pos) + } + for (; i < payload.length; i++) { + const m = i % 4 + const k = (maskingKey >>> (m === 0 ? 24 : m === 1 ? 16 : m === 2 ? 8 : 0)) & 0xFF + pos = frame.writeUint8(payload.readUint8(i) ^ k, pos) + } + } else { + frame.set(payload, pos) + pos += payload.length + } + + return frame +} + +interface Frame { + op: Opcode + fin: boolean + payload: Buffer + + // Only relevant for partial frames + maskingKey?: number + remainingLength?: number +} + +function readFrame(buf: Buffer): Frame { + let pos = 0 + const h1 = buf.readUint8(pos) + pos += 1 + const h2 = buf.readUint8(pos) + pos += 1 + + const fin = (h1 & 0x80) === 0x80 + const ext = h1 & 0x70 + const op = h1 & 0x0F + const mask = (h2 & 0x80) === 0x80 + const p1 = h2 & 0x7F + + let len: bigint | number = p1 + if (len === 126) { + len = buf.readUInt16BE(pos) + pos += 2 + } else if (len === 127) { + len = buf.readBigUint64BE(pos) + pos += 8 + if (len > Number.MAX_SAFE_INTEGER) { + throw new Error('Payload too big') + } + len = Number(len) + } + + let maskingKey: number | undefined + if (mask) { + maskingKey = buf.readUint32BE(pos) + pos += 4 + } + + const [payload, remainingLength] = readFrameIntoPayload(buf, len, pos, maskingKey) + + return { + fin, + payload, + maskingKey, + remainingLength, + op: op as Opcode, + } +} + +function rotateKey(maskingKey: number, offset: number) { + switch (offset % 4) { + case 0: + return maskingKey + case 1: + return (maskingKey >>> 8) | (maskingKey << 24) + case 2: + return (maskingKey >>> 16) | (maskingKey << 16) + case 3: + return (maskingKey >>> 24) | (maskingKey << 8) + default: + throw new Error(`Unexpected offset: ${offset}`) + } +} + +function readFrameIntoPayload(buf: Buffer, len: number, pos: number, maskingKey?: number, payload = Buffer.allocUnsafe(len), offset = 0) { + let remainingLength: number | undefined + + if (len > (buf.length - pos)) { + const truncated = buf.length - pos + remainingLength = len - truncated + len = truncated + } + + if (maskingKey !== undefined) { + maskingKey = rotateKey(maskingKey, offset) + + let i: number + for (i = 0; i < len - 4; i += 4) { + // Bitwise XOR converts everything to 32-bit integers + payload.writeInt32BE(buf.readInt32BE(pos + i) ^ maskingKey, offset + i) + } + for (; i < len; i++) { + const m = i % 4 + const k = (maskingKey >>> (m === 0 ? 24 : m === 1 ? 16 : m === 2 ? 8 : 0)) & 0xFF + payload.writeUint8(buf.readUint8(pos + i) ^ k, offset + i) + } + } else { + payload.set(Uint8Array.prototype.slice.call(buf, pos, pos + len), offset) + } + + return [payload, remainingLength] as const +} + +enum Opcode { + Continue = 0, + Text = 1, + Binary = 2, + Close = 8, + Ping = 9, + Pong = 10, +} + +enum StatusCode { + Success = 1000, + Leaving = 1001, + ProtocolError = 1002, + UnrecognizedDataType = 1003, + NoStatusCode = 1005, // MUST NOT be sent over the wire + UnexpectedCloseState = 1006, // MUST NOT be sent over the wire + InconsistentDataType = 1007, + Rejection = 1008, + MessageTooBig = 1009, + MissingExtension = 1010, + InternalError = 1011, + BadCert = 1015, +} + + +// interface EventBase { +// readonly type: string +// } + + +// interface EventTarget { +// addEventListener(type: string, callback: EventListenerOrEventListenerObject | null, options?: AddEventListenerOptions | boolean): void; +// dispatchEvent(event: EventBase): boolean; +// removeEventListener(type: string, callback: EventListenerOrEventListenerObject | null, options?: EventListenerOptions | boolean): void; +// } + +// declare var WebSocket: { +// prototype: WebSocket; +// new(url: string | URL, protocols?: string | string[]): WebSocket; +// readonly CONNECTING: 0; +// readonly OPEN: 1; +// readonly CLOSING: 2; +// readonly CLOSED: 3; +// }; + +// interface MessageEvent extends EventBase { +// readonly type: 'message' +// readonly data: T; +// /** +// * Returns the last event ID string, for server-sent events. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/lastEventId) +// */ +// readonly lastEventId: string; +// /** +// * Returns the origin of the message, for server-sent events and cross-document messaging. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/origin) +// */ +// readonly origin: string; +// /** +// * Returns the MessagePort array sent with the message, for cross-document messaging and channel messaging. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/ports) +// */ +// readonly ports: ReadonlyArray; +// /** +// * Returns the WindowProxy of the source window, for cross-document messaging, and the MessagePort being attached, in the connect event fired at SharedWorkerGlobalScope objects. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/source) +// */ +// readonly source: MessageEventSource | null; +// } + +// interface CloseEvent extends EventBase { +// readonly type: 'close' +// readonly code: number +// readonly reason: string +// readonly wasClean: boolean +// } + +// interface WebSocketEventMap { +// "close": CloseEvent; +// "error": Event; +// "message": MessageEvent; +// "open": Event; +// } + +// /** +// * Provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) +// */ +// interface WebSocket extends EventTarget { +// /** +// * Returns a string that indicates how binary data from the WebSocket object is exposed to scripts: +// * +// * Can be set, to change how binary data is returned. The default is "blob". +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/binaryType) +// */ +// binaryType: BinaryType; +// /** +// * Returns the number of bytes of application data (UTF-8 text and binary data) that have been queued using send() but not yet been transmitted to the network. +// * +// * If the WebSocket connection is closed, this attribute's value will only increase with each call to the send() method. (The number does not reset to zero once the connection closes.) +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/bufferedAmount) +// */ +// readonly bufferedAmount: number; +// /** +// * Returns the extensions selected by the server, if any. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/extensions) +// */ +// readonly extensions: string; +// /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/close_event) */ +// onclose: ((this: WebSocket, ev: CloseEvent) => any) | null; +// /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/error_event) */ +// onerror: ((this: WebSocket, ev: Event) => any) | null; +// /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/message_event) */ +// onmessage: ((this: WebSocket, ev: MessageEvent) => any) | null; +// /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/open_event) */ +// onopen: ((this: WebSocket, ev: Event) => any) | null; +// /** +// * Returns the subprotocol selected by the server, if any. It can be used in conjunction with the array form of the constructor's second argument to perform subprotocol negotiation. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/protocol) +// */ +// readonly protocol: string; +// /** +// * Returns the state of the WebSocket object's connection. It can have the values described below. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/readyState) +// */ +// readonly readyState: number; +// /** +// * Returns the URL that was used to establish the WebSocket connection. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/url) +// */ +// readonly url: string; +// /** +// * Closes the WebSocket connection, optionally using code as the the WebSocket connection close code and reason as the the WebSocket connection close reason. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/close) +// */ +// close(code?: number, reason?: string): void; +// /** +// * Transmits data using the WebSocket connection. data can be a string, a Blob, an ArrayBuffer, or an ArrayBufferView. +// * +// * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/send) +// */ +// send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void; +// readonly CONNECTING: 0; +// readonly OPEN: 1; +// readonly CLOSING: 2; +// readonly CLOSED: 3; +// addEventListener(type: K, listener: (this: WebSocket, ev: WebSocketEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; +// addEventListener(type: string, listener: EventListenerOrEventListenerObject, options?: boolean | AddEventListenerOptions): void; +// removeEventListener(type: K, listener: (this: WebSocket, ev: WebSocketEventMap[K]) => any, options?: boolean | EventListenerOptions): void; +// removeEventListener(type: string, listener: EventListenerOrEventListenerObject, options?: boolean | EventListenerOptions): void; +// } + +// declare var WebSocket: { +// prototype: WebSocket; +// new(url: string | URL, protocols?: string | string[]): WebSocket; +// readonly CONNECTING: 0; +// readonly OPEN: 1; +// readonly CLOSING: 2; +// readonly CLOSED: 3; +// }; diff --git a/src/runtime/nodeLoader.ts b/src/runtime/nodeLoader.ts new file mode 100644 index 0000000..4a7da88 --- /dev/null +++ b/src/runtime/nodeLoader.ts @@ -0,0 +1,169 @@ +import * as fs from 'node:fs' +import * as path from 'node:path' +import { createModuleResolver } from './resolver' +import { BasicDataRepository, createModuleLoader } from './loader' +import { createCodeCache } from './utils' +import { getV8CacheDirectory } from '../workspaces' +import { resolveValue } from './modules/serdes' +import { ImportMap, SourceInfo } from './importMaps' +import { throwIfNotFileNotFoundError } from '../utils' +import { openBlock } from '../build-fs/block' +import { homedir } from 'node:os' +import { getDataRepository } from '../artifacts' +import { getFs, setContext } from '../execution' + +async function findImportMap(dir: string): Promise | undefined> { + const fileName = path.resolve(dir, 'import-map.json') + + try { + return JSON.parse(await fs.promises.readFile(fileName, 'utf-8')) + } catch (e) { + throwIfNotFileNotFoundError(e) + + const next = path.dirname(dir) + if (next !== dir) { + return findImportMap(next) + } + } +} + +async function findDataDir(dir: string): Promise { + const dataDir = path.resolve(dir, '.synapse', 'build', 'data') + if (await fs.promises.access(dataDir, fs.constants.F_OK).then(() => true, () => false)) { + return dataDir + } + + const next = path.dirname(dir) + if (next !== dir) { + return findDataDir(next) + } +} + +async function getImportMap(targetPath: string): Promise | undefined> { + const importMapFilePath = process.env['JS_IMPORT_MAP_FILEPATH'] + if (!importMapFilePath) { + return findImportMap(path.dirname(targetPath)) + } + + return JSON.parse(await fs.promises.readFile(importMapFilePath, 'utf-8')) +} + +function tryReadFile(fileName: string) { + try { + return fs.readFileSync(fileName) + } catch (e) { + throwIfNotFileNotFoundError(e) + } +} + +const getSynapseInstallDir = () => process.env['SYNAPSE_INSTALL'] || path.resolve(homedir(), '.synapse') +const getGlobalBuildDir = () => path.resolve(getSynapseInstallDir(), 'build') + +function findDataRepoSync(dir: string): { dataDir: string; repo: BasicDataRepository } | undefined { + const fileName = path.resolve(dir, '.synapse', 'snapshot.json') + const d = tryReadFile(fileName)?.toString('utf-8') + if (d) { + const hash = JSON.parse(d).storeHash as string + const blockFile = path.resolve(dir, '.synapse', 'blocks', hash) + const b = tryReadFile(blockFile) + if (b) { + const block = openBlock(b) + + // XXX: these lines + `_getDataSync` only exist to support loading deployed package deps + const fs2 = getFs() + setContext({ id: 'dev', fs: fs2 }) + const repo = getDataRepository(fs2, getGlobalBuildDir()) + + function _getDataSync(hash: string) { + try { + return block.readObject(hash) + } catch (e) { + if (!(e as any).message?.includes('Object not found')) { + throw e + } + + return Buffer.from(repo.readDataSync(hash)) + } + } + + function getDataSync(hash: string): Buffer + function getDataSync(hash: string, encoding: BufferEncoding): string + function getDataSync(hash: string, encoding?: BufferEncoding) { + if (!encoding) { + return _getDataSync(hash) + } + return _getDataSync(hash).toString(encoding) + } + + return { + dataDir: path.resolve(dir, '.synapse', 'data'), + repo: { getDataSync } + } + } + } + + const next = path.dirname(dir) + if (next !== dir) { + return findDataRepoSync(next) + } +} + +export async function devLoader(target: string) { + const workingDirectory = process.cwd() + const resolvedPath = await fs.promises.realpath(path.resolve(workingDirectory, target)) + const targetDir = path.dirname(resolvedPath) + const importMap = await getImportMap(targetDir) + + const resolver = createModuleResolver({ + readFileSync: fs.readFileSync, + fileExistsSync: (fileName: string) => { + try { + fs.accessSync(fileName) + return true + } catch { + return false + } + } + }, workingDirectory) + + if (importMap) { + resolver.registerMapping(importMap) + + // XXX: the import map doesn't associate globals with pointers + const globals: Record = {} + for (const [k, v] of Object.entries(importMap)) { + if (k.startsWith('synapse:')) { + globals[k] = v.location + } + } + + resolver.registerGlobals(globals) + } + + const codeCachePath = getV8CacheDirectory() + const codeCache = createCodeCache({ + readFileSync: fs.readFileSync, + deleteFileSync: (p) => fs.rmSync(p), + writeFileSync: (p, data) => { + try { + fs.writeFileSync(p, data) + } catch (e) { + throwIfNotFileNotFoundError(e) + fs.mkdirSync(codeCachePath, { recursive: true }) + fs.writeFileSync(p, data) + } + }, + }, codeCachePath) + + const found = findDataRepoSync(targetDir) + const dataDir = found?.dataDir ?? (await findDataDir(targetDir) ?? await findDataDir(workingDirectory) ?? workingDirectory) + const loader = createModuleLoader(fs, dataDir, resolver, { + codeCache, + deserializer: resolveValue, + dataRepository: found?.repo, + useThisContext: true, + }) + + return loader()(resolvedPath) +} + diff --git a/src/runtime/resolver.ts b/src/runtime/resolver.ts new file mode 100644 index 0000000..263d2ad --- /dev/null +++ b/src/runtime/resolver.ts @@ -0,0 +1,513 @@ +import * as path from 'node:path' +import { isBuiltin } from 'node:module' +import { SyncFs } from '../system' +import { getSpecifierComponents, resolveBareSpecifier, resolvePrivateImport } from '../pm/packages' +import { createTrie, isRelativeSpecifier, keyedMemoize, throwIfNotFileNotFoundError } from '../utils' +import { ImportMap, SourceInfo } from './importMaps' +import { PackageJson } from '../pm/packageJson' +import { isDataPointer, toAbsolute } from '../build-fs/pointers' + +const pointerPrefix = 'pointer:' +export const synapsePrefix = 'synapse:' +export const providerPrefix = 'synapse-provider:' + +type LocationType = 'module' | 'package' + +interface Mapping { + readonly virtualLocation: string + readonly physicalLocation: string + readonly locationType?: LocationType +} + +export interface MapNode { + readonly location: string + readonly locationType?: LocationType + readonly mappings: Record + readonly source?: T +} + +function createLookupTable() { + const trie = createTrie, string[]>() + const keys = new Map() + + function getMapKey(map: ImportMap[string]) { + if (keys.has(map)) { + return keys.get(map)! + } + + const k = `__vp${keys.size}` + keys.set(map, k) + + return k + } + + const locationKeys = new Map() + function getLocationKey(location: string): string[] { + const cached = locationKeys.get(location) + if (cached !== undefined) { + return cached + } + + const segments = location.split(path.sep) + const trimmed = location.endsWith(path.sep) ? segments.slice(-1) : segments + locationKeys.set(location, trimmed) + + return trimmed + } + + function updateNode(key: string[], map: ImportMap, location: string, rootKey = key, locationType?: LocationType, source?: T, visited = new Set>) { + if (visited.has(map)) return + visited.add(map) + + const mappings = trie.get(key)?.mappings ?? {} + trie.insert(key, { location, locationType, mappings, source }) + + for (const [k, v] of Object.entries(map)) { + const ck = getMapKey(v) + const key = [...rootKey, ck] + const virtualLocation = key.join(path.sep) + locationKeys.set(virtualLocation, key) + mappings[k] = { + virtualLocation, + physicalLocation: v.location, + locationType: v.locationType, + } + + // Sift the mapping upwards to simulate node + if (key !== rootKey) { + const m = trie.get(rootKey)?.mappings + if (m && !m[k]) { + m[k] = mappings[k] + } + } + + updateNode(key, v.mapping ?? {}, v.location, rootKey, v.locationType, v.source, new Set([...visited])) + } + } + + function lookup(specifier: string, location: string): Mapping | undefined { + const key = getLocationKey(location) + const stack: (readonly [string, MapNode | undefined])[] = [] + for (const n of trie.traverse(key)) { + stack.push(n) + } + + while (stack.length > 0) { + const [k, m] = stack.pop()! + const v = m?.mappings[specifier] + if (v) { + return v + } + } + } + + function resolve(location: string | Mapping) { + if (typeof location === 'object' && 'physicalLocation' in location) { + return location.physicalLocation + } + + const key = getLocationKey(location) + const stack: (MapNode | undefined)[] = [] + for (const [k, v] of trie.traverse(key)) { + stack.push(v) + } + + const last = stack.pop() + if (!last || last.location === '/') { + return location + } + + // We add 1 because we popped the stack + const suffix = key.slice(stack.length + 1) + if (last.locationType === 'module' && suffix.length > 0) { + return [path.dirname(last.location), ...suffix].join(path.sep) + } + + return [last.location, ...suffix].join(path.sep) + } + + function registerMapping(map: ImportMap, location: string = '/') { + const key = getLocationKey(location) + updateNode(key, map, location) + } + + function getSource(location: string) { + const key = getLocationKey(location) + const stack: (MapNode | undefined)[] = [] + for (const [k, v] of trie.traverse(key)) { + stack.push(v) + } + + return stack[stack.length - 1] + } + + function inspect() { + function visit(key?: string[], value?: MapNode, depth = 0) { + for (const k of trie.keys(key)) { + const nk = key ? [...key, k] : [k] + const value = trie.get(nk) + console.log(`${' '.repeat(depth + 1)} -- ${k} -- ${value?.location}`) + + visit(nk, value, depth + 1) + } + } + + visit() + } + + return { lookup, resolve, registerMapping, getSource, inspect } +} + +export interface PatchedPackage { + readonly name: string + // readonly version?: string + readonly files: Record string> +} + +export type ModuleTypeHint = 'cjs' | 'esm' | 'native' | 'pointer' | 'builtin' | 'json' | 'sea-asset' + +// Module resolution has two phases: +// 1. Determine the _virtual_ file that a specifier + location maps to +// 2. Resolve the virtual file to a location on disk to get its contents + +// Virtual files represent a specific instantiation of a module. Two files may have the +// exact same source code but have different import maps. So they must be treated as +// separate entities. + +export type ModuleResolver = ReturnType +export function createModuleResolver(fs: Pick, workingDirectory: string) { + const lookupTable = createLookupTable() + const globals: Record = {} + const resolvedSubpaths = new Map() + + const getPackage = keyedMemoize(function (dir: string): { data: PackageJson, filePath: string } { + const filePath = path.resolve(dir, 'package.json') + const data = JSON.parse(fs.readFileSync(filePath, 'utf-8')) + + return { data, filePath } + }) + + const packageMap = new Map() + function getCurrentPackage(location: string) { + try { + return getPackage(location) + } catch (e) { + throwIfNotFileNotFoundError(e) + + const next = path.dirname(location) + if (next !== location) { + return getCurrentPackage(next) + } + + throw e + } + } + + function resolveProvider(specifier: string, importer: string) { + const res = lookupTable.lookup(specifier, importer) + if (!res) { + throw new Error(`Failed to resolve provider: ${specifier} [${importer ?? workingDirectory}]`) + } + + const resolved = lookupTable.resolve(res) + const pkg = path.resolve(resolved, 'package.json') + const pkgData = JSON.parse(fs.readFileSync(pkg, 'utf-8')) + + return { + location: resolved, + module: path.resolve(resolved, pkgData.exports['.']), + name: pkgData.name, + source: pkgData.source, + version: pkgData.version, + } + } + + function resolveRelative(specifier: string, location: string) { + const extname = path.extname(specifier) + const absPath = path.resolve(location, specifier) + + if (extname === '.js' || extname === '.json') { // TODO: .cjs/.mjs ?? + return absPath + } + + const candidates = [ + path.resolve(absPath, 'index.js'), + path.resolve(absPath, 'index.json'), + absPath + ] + + if (specifier !== '.') { + candidates.unshift(`${absPath}.infra.js`) // XXX: needed when not compiling with `--include-js` + candidates.unshift(`${absPath}.json`) + candidates.unshift(`${absPath}.js`) + // candidates.unshift(`${absPath}.node`) + } + + for (const p of candidates) { + if (fs.fileExistsSync(p)) { + // handles relative self-referential imports + if (p === absPath && !extname) { + try { + const pkg = getPackage(absPath) + if (pkg.data.main) { + return path.resolve(absPath, pkg.data.main) + } + } catch {} + } + return p + } + } + + throw new Error(`Failed to resolve module: ${specifier} [importer: ${location}]`) + } + + function resolveWorker(specifier: string, importer?: string, mode: 'cjs' | 'esm' = 'cjs'): string | [fileName: string, typeHint: ModuleTypeHint] { + if (isBuiltin(specifier)) { + return specifier + } + + if (globals[specifier]) { + return specifier + } + + if (specifier.startsWith(pointerPrefix)) { + const res = lookupTable.lookup(specifier, importer ?? workingDirectory) + if (res !== undefined) { + return res.virtualLocation + } + + if (isDataPointer(specifier)) { + return [specifier, 'pointer'] + } + + return specifier + } + + const getLocation = () => importer ? path.dirname(lookupTable.resolve(importer)) : workingDirectory + + if (specifier.startsWith(providerPrefix)) { + // FIXME: need to call `registerPointerDependencies` from `dynamicImport` to remove the fallback + const res = lookupTable.lookup(specifier, importer ?? workingDirectory) ?? lookupTable.lookup(specifier, workingDirectory) + if (!res) { + throw new Error(`Failed to resolve provider: ${specifier} [${importer ?? workingDirectory}]`) + } + + const resolved = lookupTable.resolve(res) + const pkg = path.resolve(resolved, 'package.json') + const pkgData = JSON.parse(fs.readFileSync(pkg, 'utf-8')) + + return path.resolve(resolved, pkgData.exports['.']) + } + + if (isRelativeSpecifier(specifier)) { + if (!importer) { + return resolveRelative(specifier, getLocation()) + } + + if (globals[importer]) { + return resolveRelative(specifier, path.dirname(globals[importer])) + } + + const resolvedImporter = lookupTable.resolve(importer) + const filePath = path.dirname(resolvedImporter) + const res = resolveRelative(specifier, filePath) + const rel = path.relative(filePath, res) + + return path.join(path.dirname(importer), rel) + } + + const components = getSpecifierComponents(specifier) + const key = components.scheme ? `${components.scheme}:${components.name}` : components.name + const res = lookupTable.lookup(key, importer ?? workingDirectory) + if (res !== undefined) { + const filePath = lookupTable.resolve(res) + if (specifier.startsWith(synapsePrefix)) { + return filePath + } + + const pkg = getPackage(filePath) + packageMap.set(pkg.filePath, res.virtualLocation) // Only used for private subpath imports + resolvePatches(pkg) + + const rel = resolveBareSpecifier(specifier, pkg.data, mode) + const absPath = resolveRelative(rel.fileName, filePath) + const virtualId = path.join(res.virtualLocation, path.relative(filePath, absPath)) + if (components.export) { + resolvedSubpaths.set(virtualId, components.export) + } + + return [virtualId, rel.moduleType] + } + + if (specifier[0] === '#') { + const filePath = lookupTable.resolve(getLocation()) + const pkg = getCurrentPackage(filePath) + const rel = resolvePrivateImport(specifier, pkg.data, mode) + const absPath = resolveRelative(rel.fileName, filePath) + const virtualId = path.join(packageMap.get(pkg.filePath) ?? path.dirname(pkg.filePath), path.relative(filePath, absPath)) + + return [virtualId, rel.moduleType] + } + + if (path.isAbsolute(specifier)) { + return specifier + } + + // FIXME: need to call `registerPointerDependencies` from `dynamicImport` to remove the fallback + if (importer?.startsWith(pointerPrefix)) { + return nodeModulesResolve(workingDirectory, specifier, components, mode) + // throw new Error(`Failed to resolve module: ${specifier} [${importer}]`) + } + + return nodeModulesResolve(getLocation(), specifier, components, mode) + } + + const pkgCache: Record> = {} + function findPkg(dir: string, name: string) { + const key = `${dir}:${name}` + const cached = pkgCache[key] + if (cached) { + return cached + } + + const pkgPath = path.resolve(dir, 'node_modules', name) + + try { + return pkgCache[key] = getPackage(pkgPath) + } catch (e) { + throwIfNotFileNotFoundError(e) + + const next = path.dirname(dir) + if (next !== dir) { + return findPkg(next, name) + } + + // TODO: add stack + throw new Error(`Failed to resolve package: ${name}`) + } + } + + function nodeModulesResolve(dir: string, specifier: string, components: ReturnType, mode: 'cjs' | 'esm' = 'cjs'): [fileName: string, typeHint: ModuleTypeHint] { + const pkg = findPkg(dir, components.name) + const rel = resolveBareSpecifier(specifier, pkg.data, mode) + const absPath = resolveRelative(rel.fileName, path.dirname(pkg.filePath)) + + return [absPath, rel.moduleType] + } + + function resolveVirtual(specifier: string, importer?: string, mode: 'cjs' | 'esm' = 'cjs') { + try { + const resolved = resolveWorker(specifier, importer, mode) + + return typeof resolved === 'string' ? resolved : resolved[0] + } catch (e) { + throw Object.assign( + new Error(`Failed to resolve ${specifier} from ${importer} [mode: ${mode}]`, { cause: e }), + { code: 'MODULE_NOT_FOUND' } // node compat + ) + } + } + + function resolveVirtualWithHint(specifier: string, importer?: string, mode: 'cjs' | 'esm' = 'cjs') { + try { + return resolveWorker(specifier, importer, mode) + } catch (e) { + throw Object.assign( + new Error(`Failed to resolve ${specifier} from ${importer} [mode: ${mode}]`, { cause: e }), + { code: 'MODULE_NOT_FOUND' } // node compat + ) + } + } + + function getFilePath(id: string) { + if (globals[id]) { + return globals[id] + } + + return lookupTable.resolve(id) + } + + function resolve(specifier: string, importer?: string) { + return getFilePath(resolveVirtual(specifier, importer)) + } + + const patches = new Map() + const resolved = new Set() + const resolvedPatches = new Map() + function getPatchFn(resolvedPath: string) { + return resolvedPatches.get(resolvedPath) + } + + function registerPatch(patch: PatchedPackage) { + if (patches.has(patch.name)) { + throw new Error(`Patch already registered for package: ${patch.name}`) + } + + patches.set(patch.name, patch) + } + + function resolvePatches(pkg: ReturnType) { + const patch = patches.get(pkg.data.name) + if (!patch || resolved.has(pkg.filePath)) { + return + } + + for (const [k, v] of Object.entries(patch.files)) { + const resolved = path.resolve(path.dirname(pkg.filePath), k) + resolvedPatches.set(resolved, v) + } + + resolved.add(pkg.filePath) + } + + function registerGlobals(mapping: Record) { + for (const [k, v] of Object.entries(mapping)) { + if (globals[k] && globals[k] !== v) { + throw new Error(`Conflicting global module mapping for "${k}": ${v} [new] !== ${globals[k]} [existing]`) + } + globals[k] = v + } + } + + function getInverseGlobalsMap() { + return Object.fromEntries(Object.entries(globals).map(e => e.reverse())) as Record + } + + function getSource(location: string): (MapNode & { subpath?: string }) | undefined { + const source = lookupTable.getSource(location) + if (!source) { + return + } + + const subpath = resolvedSubpaths.get(location) + if (subpath) { + return Object.assign({ subpath }, source) + } + + return source + } + + return { + resolve, + resolveVirtual, + resolveVirtualWithHint, + resolveProvider, + getFilePath, + registerMapping: lookupTable.registerMapping, + + // PATCHING + registerPatch, + getPatchFn, + + // GLOBAL SPECIFIERS + registerGlobals, + getInverseGlobalsMap, + + // MISC + getSource, + } +} + +export function createImportMap(map: Record, prefix: string = ''): ImportMap { + return Object.fromEntries(Object.entries(map).map(([k, v]) => [`${prefix}${k}`, { location: v, versionConstraint: '*' }] as const)) +} \ No newline at end of file diff --git a/src/runtime/rootLoader.ts b/src/runtime/rootLoader.ts new file mode 100644 index 0000000..fecb4a7 --- /dev/null +++ b/src/runtime/rootLoader.ts @@ -0,0 +1,89 @@ +import * as esbuild from 'esbuild' +import { readFileSync, existsSync } from 'node:fs' +import { DataRepository, getDataRepository } from '../artifacts' +import { getBuildTarget, getFs } from '../execution' +import { Fs, SyncFs } from '../system' +import { getV8CacheDirectory } from '../workspaces' +import { createModuleLoader, BasicDataRepository } from './loader' +import { ModuleResolver, createModuleResolver } from './resolver' +import { createCodeCache } from './utils' +import { setupEsbuild } from '../bundler' + +function toBuffer(arr: Uint8Array): Buffer { + return Buffer.isBuffer(arr) ? arr : Buffer.from(arr) +} + +export function createBasicDataRepo(repo: DataRepository): BasicDataRepository { + function getDataSync(hash: string): Buffer + function getDataSync(hash: string, encoding: BufferEncoding): string + function getDataSync(hash: string, encoding?: BufferEncoding) { + const data = toBuffer(repo.readDataSync(hash)) + + return encoding ? data.toString(encoding) : data + } + + return { getDataSync } +} + +export function createModuleResolverForBundling(fs: Fs & SyncFs, workingDirectory: string): ModuleResolver { + const resolver = createModuleResolver(fs, workingDirectory) + + // Need to patch this file because it's not compatible w/ bundling to ESM + resolver.registerPatch({ + name: '@aws-crypto/util', + // version: 3.0.0 + files: { + 'build/convertToBuffer.js': contents => contents.replace('require("@aws-sdk/util-utf8-browser")', '{}') + } + }) + + return resolver +} + +function loadEsbuildWithWorkersDisabled() { + process.env['ESBUILD_WORKER_THREADS'] = '0' + setupEsbuild() + delete process.env['ESBUILD_WORKER_THREADS'] +} + +function createTypescriptLoader() { + return (fileName: string, format: 'cjs' | 'esm' = 'cjs') => { + loadEsbuildWithWorkersDisabled() + + // TODO: add option to configure sourcemap + const contents = readFileSync(fileName) + const res = esbuild.transformSync(contents, { format, loader: 'ts', sourcemap: 'inline' }) + + return res.code + } +} + +export function createMinimalLoader(useTsLoader = false) { + const bt = getBuildTarget() + const workingDirectory = bt?.workingDirectory ?? process.cwd() + const codeCache = createCodeCache(getFs(), getV8CacheDirectory()) + const typescriptLoader = useTsLoader ? createTypescriptLoader() : undefined + + const loader = createModuleLoader( + { readFileSync }, + workingDirectory, + createModuleResolver({ readFileSync, fileExistsSync: existsSync }, workingDirectory), + { + codeCache, + workingDirectory, + typescriptLoader, + dataRepository: bt ? createBasicDataRepo(getDataRepository(getFs(), bt.buildDir)) : undefined, + useThisContext: true, + } + ) + + function loadModule(id: string, origin?: string) { + if (id.endsWith('.mjs')) { + return loader.loadEsm(id, origin) + } + + return loader(origin)(id) + } + + return { loadModule } +} \ No newline at end of file diff --git a/src/runtime/sourceMaps.ts b/src/runtime/sourceMaps.ts new file mode 100644 index 0000000..1934004 --- /dev/null +++ b/src/runtime/sourceMaps.ts @@ -0,0 +1,327 @@ +import { getLogger } from '../logging' +import { isNonNullable } from '../utils' + +const base64Alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' +function encodeVlqBase64(vlqs: number[]) { + return vlqs.map(n => base64Alphabet[n]).join('') +} + +const base64Map = Object.fromEntries(base64Alphabet.split('').map((c, i) => [c, i])) + +function decodeVlqBase64(vlq: string) { + const result: number[] = [] + for (let i = 0; i < vlq.length; i++) { + result.push(base64Map[vlq[i]]) + } + return result +} + +export function encodeVlq(n: number): number[] { + if (n === 0) return [n] + + let val = Math.abs(n) + const result: number[] = [] + while (val > 0) { + let x = 0 + if (result.length === 0) { + x |= (val & 0xF) << 1 + x |= n >= 0 ? 0 : 1 + val >>>= 4 + } else { + x = val & 0x1F + val >>>= 5 + } + + x |= val > 0 ? 0x20 : 0 + result.push(x) + } + + return result +} + +export function decodeVlq(vlq: number[]): number[] { + const result: number[] = [] + let val: number | undefined = undefined + let isNegative = false + let offset = 0 + for (let i = 0; i < vlq.length; i++) { + if (val === undefined) { + val = (vlq[i] >>> 1) & 0xF + isNegative = (vlq[i] & 0x1) === 1 + offset = 4 + } else { + val |= (vlq[i] & 0x1F) << offset + offset += 5 + } + + if ((vlq[i] & 0x20) === 0) { + result.push(isNegative ? -val : val) + val = undefined + offset = 0 + } + } + + if (val !== undefined) { + throw new Error(`Malformed VLQ array did not terminate: ${vlq}`) + } + + return result +} + +export function decodeMappings(mappings: string) { + let prevSourceIdx = 0, prevSourceLine = 0, prevSourceColumn = 0, prevNameIdx = 0 + + const res: MappingSegment[] = [] + const lines = mappings.split(';') + for (let i = 0; i < lines.length; i++) { + let prevColumn = 0 + const segments = lines[i].split(',') + for (let j = 0; j < segments.length; j++) { + const segment = segments[j] + if (!segment) continue + + const decoded = decodeVlq(decodeVlqBase64(segment)) + if (decoded.length === 0) { + throw new Error(`Malformed source mapping: ${segment}`) + } + + const fromColumn = decoded[0] + prevColumn + prevColumn = fromColumn + + if (decoded.length === 1) { + res.push({ + fromLine: i, + toLine: i, + fromColumn, + toColumn: 0, // Not sure if these mappings are ever relevant + }) + } else if (decoded.length >= 4) { + const sourceIndex = decoded[1] + prevSourceIdx + const toLine = decoded[2] + prevSourceLine + const toColumn = decoded[3] + prevSourceColumn + const symbolIndex = decoded.length >= 5 ? decoded[4] + prevNameIdx : undefined + prevSourceIdx = sourceIndex + prevSourceLine = toLine + prevSourceColumn = toColumn + prevNameIdx = symbolIndex ?? prevNameIdx + + res.push({ + fromLine: i, + toLine, + fromColumn, + toColumn, + sourceIndex, + symbolIndex, + }) + } + } + } + + return res +} + +export interface SourceMapV3 { + version: 3 + file?: string + sourceRoot?: string + sources: string[] + sourcesContent?: string[] + names: string[] + mappings: string +} + +// from = generated +// to = original + +interface Mapping { + fromLine: number // zero-based + toLine: number + fromColumn: number + toColumn: number +} + +interface MappingSegment extends Mapping { + sourceIndex?: number + symbolIndex?: number +} + +export function toInline(sourcemap: SourceMapV3) { + const encoded = Buffer.from(JSON.stringify(sourcemap), 'utf-8').toString('base64') + + return `//# sourceMappingURL=data:application/json;base64,${encoded}` +} + +export function createSourceMapGenerator() { + const groups = new Map() + + function addMapping(mapping: Mapping) { + const line = mapping.fromLine + if (!groups.has(line)) { + groups.set(line, []) + } + + const group = groups.get(mapping.fromLine)! + group.push(mapping) + } + + function generate(sourceFileName: string): SourceMapV3 { + const lines = Array.from(groups.keys()).sort() + if (lines.length === 0) { + return { version: 3, sources: [sourceFileName], names: [], mappings: '' } + } + + const lastLine = lines[lines.length - 1] + + let mappings = '' + let prevSourceIdx = 0, prevSourceLine = 0, prevSourceColumn = 0, prevNameIdx = 0 + for (let i = 0; i < lastLine; i++) { + const g = groups.get(i) + if (!g) { + mappings += ';' + continue + } + + let prevColumn = 0 + const segs: string[] = [] + for (const segment of g) { + const column = segment.fromColumn - prevColumn + const sourceLine = segment.toLine - prevSourceLine + const sourceColumn = segment.toColumn - prevSourceColumn + const vlqs = [ + ...encodeVlq(column), + 0, // sourceIdx + ...encodeVlq(sourceLine), + ...encodeVlq(sourceColumn), + ] + + prevColumn = segment.fromColumn + prevSourceLine = segment.toLine + prevSourceColumn = segment.toColumn + segs.push(`${encodeVlqBase64(vlqs)}`) + } + mappings += `${segs.join(',')};` + } + + return { + version: 3, + sources: [sourceFileName], + names: [], + mappings, + } + } + + return { addMapping, generate } +} + +export function parseDirectives(lines: string[]) { + const directives = lines + .map(l => l.match(/^\/\/#\s*([\w]+)\s*=\s*([^\s]+)/)) + .filter(isNonNullable) + .map(c => [c[1], c[2]] as const) + + return Object.fromEntries(directives) +} + +function findFooterDirective(contents: string) { + const idx = contents.lastIndexOf('//#') + if (idx === -1) { + return + } + + return contents.slice(idx + 3).trim() +} + +interface InlineResult { + readonly type: 'inline' + readonly data: SourceMapV3 +} + +interface ReferenceResult { + readonly type: 'reference' + readonly location: string +} + +type ParseResult = InlineResult | ReferenceResult + +function parseSourceMap(directive: string): ParseResult | undefined { + const url = /^sourceMappingURL=(.*)/.exec(directive)?.[1] + if (!url) { + return + } + + if (url.startsWith('data:')) { + const [type, data] = url.slice(5).split(',', 2) + const [contentType, encoding = 'utf-8'] = type.split(';', 2) + if (contentType !== 'application/json') { + getLogger().log('Not implemented:', directive) + return + } + + if (encoding !== 'utf-8' && encoding !== 'base64') { + getLogger().log('Not implemented:', directive) + return + } + + const parsed = JSON.parse(Buffer.from(data, 'base64').toString('utf-8')) as SourceMapV3 + + return { type: 'inline', data: parsed } + } + + return { type: 'reference', location: url } +} + +export function findSourceMap(fileName: string, contents: string) { + const directive = findFooterDirective(contents) + if (!directive) { + return + } + + const result = parseSourceMap(directive) + if (result?.type === 'inline') { + return result.data + } else if (result?.type === 'reference') { + const baseUrl = new URL(`file:${fileName}`) + + return new URL(result.location, baseUrl) + } +} + +export function dumpMappings(sourcemap: SourceMapV3) { + const mappings = decodeMappings(sourcemap.mappings) + + for (const m of mappings) { + console.log(`${m.fromLine}:${m.fromColumn} -> ${m.toLine}:${m.toColumn}`) + } +} + +export function mergeSourcemaps(oldMap: SourceMapV3, newMap: SourceMapV3): SourceMapV3 { + const merged = createSourceMapGenerator() + const oldMappings = decodeMappings(oldMap.mappings) + const index: Record = {} + + for (const m of oldMappings) { + const arr = index[m.fromLine] ??= [] + arr.push(m) + } + + function getPosition(segment: MappingSegment) { + const mapping = index[segment.toLine]?.find(m => m.fromColumn === segment.toColumn) + + if (mapping) { + return { + ...segment, + toLine: mapping.toLine, + toColumn: mapping.toColumn, + } + } + } + + for (const segment of decodeMappings(newMap.mappings)) { + const m = getPosition(segment) + if (m) { + merged.addMapping(m) + } + } + + return merged.generate(oldMap.sources[0]) +} diff --git a/src/runtime/srl/README.md b/src/runtime/srl/README.md new file mode 100644 index 0000000..9ecbc54 --- /dev/null +++ b/src/runtime/srl/README.md @@ -0,0 +1,39 @@ +## Introduction + +The Standard Resource Library (SRL) is a collection of interfaces and utility functions that abstract away services provided by cloud providers. Common interfaces facilitate creating "cloud-agnostic" applications, which can be deployed to any cloud provider. + +Having a standard right from the start, even if minimal, is intended to at least get the industry thinking about standardization. In all liklihood, standardization of cloud technology will take many years or even decades. As such, incremental standardization is a key requirement of the SRL. + + +### Balancing applicability and flexibility (WIP) + +All abstractions come with a tradeoff: the more general something becomes, the fewer "free variables" can be changed by producers or relied on by consumers. Abstractions fundamentally restrict the information space in the domain they are applied. Information restrictions impede customization; the less I know about something, the less I can change. So an ideal abstraction is one that both minimizes information restriction while maximizing the number of consumers who can reliably use the abstraction. + +For resources, this means allowing for inputs that do not fundamentally change the resulting interface. I should be able to use a resource the same way regardless of how it was created. Here are a few guidelines for creating such interfaces: + +* Prefer generic (parameterized) interfaces +* Minimize the number of methods and properties +* When using discrete outputs: + * Avoid "optional" (nullable) fields + * Keep the result of methods as simple as possible +* Avoid "convenience" methods, especially ones that create resources + * Prefering creating a separate function/resource declaration instead +* Avoid optional parameters that only change the output shape, not behavior + * Prefer creating utility functions instead. Documentation should have examples using the functions to increase visibility. + + +In practice, using a resource "the same way" has many asterisks attached. For example, a resource from one cloud provider may be able to tolerate much higher workloads compared to another one. This can cause all sorts of problems in applications that depend on specific timings or performance characteristics. + + +### Per-vendor customizations +Taking inspiration from CSS vendor parameters, all SRL resources can be customized on a per-vendor basis. The idea is that this would allow "progressive enhancement" of applications, exposing additional features to the operator. Such things might include logging, analytics, or premium features that would otherwise be undesireable in the baseline implementation. + + +### Compliance Checking +Every resource interface has a baseline test suite that can be used to test for compliance. The nature of integrations means that it is up to the _integrator_ to run these tests. In the future, compliance checking could be centralized to give users more confidence in the libraries they're using. + +These tests _only_ check that the implementation adheres to the interface; additional characteristics such as performance are not tested. Standardized performance tests are a possible future addition to the SRL. + + +### Governance (WIP) +Ideally, the SRL would be governed by an international standards organization. Until then, Cohesible will have stewardship over the SRL. \ No newline at end of file diff --git a/src/runtime/srl/compute/index.ts b/src/runtime/srl/compute/index.ts new file mode 100644 index 0000000..66012c9 --- /dev/null +++ b/src/runtime/srl/compute/index.ts @@ -0,0 +1,168 @@ +//# moduleId = synapse:srl/compute + +import { HttpHandler, HttpRequest, HttpResponse, HttpRoute, PathArgs } from 'synapse:http' +// import { WebSocket } from 'synapse:ws' +import { HostedZone, Network } from 'synapse:srl/net' + +//# resource = true +/** @internal */ +export declare class Host { + constructor(network: Network, target: () => Promise | void , key?: KeyPair) + // ssh(user: string, keyPath: string): Promise +} + +//# resource = true +/** @internal */ +export declare class KeyPair { + public constructor(publicKey: string) +} + +// XXX: `opt.createImage` and `opt.imageCommands` is an experiment +export interface FunctionOptions { + /** @internal */ + network?: Network // TODO: `Network` should be apart of the context + /** @internal */ + createImage?: boolean + /** @internal */ + imageCommands?: string[] + external?: string[] + /** @internal */ + extraFiles?: [string, string][] + /** @internal */ + memory?: number // Vendor option? + timeout?: number + /** @internal */ + reservedConcurrency?: number, // Vendor option + /** @internal */ + ephemeralStorage?: number // Vendor option +} + +//# resource = true +//# callable = invoke +export declare class Function { + constructor(fn: (...args: T) => Promise | U, opts?: FunctionOptions) + invoke(...args: T): Promise + invokeAsync(...args: T): Promise +} + +export interface Function { + (...args: T): Promise +} + +export interface HttpServiceOptions { + /** + * Sets the authorization method for all routes on the service. + * + * Can be one of: + * * `none` - no authorization + * * `native` - authorizes using the target cloud provider (the default) + * * A custom function that intercepts the request. Returning nothing passes the request on to the router. + */ + auth?: 'none' | 'native' | HttpHandler + + /** + * Uses a custom domain name instead of the one provided by the cloud provider. + */ + domain?: HostedZone + + /** @internal */ + allowedOrigins?: string[] + + /** @internal */ + mergeHandlers?: boolean +} + +//# resource = true +export declare class HttpService { + readonly hostname: string + readonly invokeUrl: string + /** @internal */ + readonly defaultPath?: string + + constructor(opt?: HttpServiceOptions) + + /** @internal */ + callOperation(route: HttpRoute, ...args: T): Promise + + /** @internal */ + forward(req: HttpRequest, body: any): Promise + + //# resource = true + /** @internal */ + addRoute( + route: U, + handler: HttpHandler, + opt: { rawBody: true } + ): HttpRoute<[...PathArgs, string], R> + + //# resource = true + addRoute

( + route: P, + handler: HttpHandler, + opt?: { + /** @internal */ + rawBody?: boolean + } + ): HttpRoute<[...PathArgs

, ...(U extends undefined ? [] : [body: U])], R> + + /** TODO */ + // addMiddleware( + // middleware: (req: HttpRequest, body: any, next: HttpHandler) => HttpResponse | Promise + // ): any +} + +// //# resource = true +// export declare class WebSocketService { +// readonly invokeUrl: string +// constructor(opt?: HttpServiceOptions) + +// addWebSocketRoute

( +// route: P, +// handler: HttpHandler, +// opt?: { rawBody?: boolean } +// ): HttpRoute<[...PathArgs

, ...(U extends undefined ? [] : [body: U])], R> +// } + +/** @internal */ +export interface ContainerInstance { + readonly name: string + readonly ip: string + readonly port: number // obviously this is 1:1 w/ a running app +} + +//# resource = true +/** @internal */ +export declare class Container { + constructor(network: Network, target: () => Promise | void) + updateTaskCount(count: number): Promise + listInstances(): Promise + //# resource = true + static fromEntrypoint(network: Network, entrypoint: () => any): Container + //# resource = true + static fromDockerfile(network: Network, dockerfile: string): Container +} + +/** @internal */ +export interface AcquiredLock extends AsyncDisposable { + readonly id: string +} + +//# resource = true +/** @internal */ +export declare class SimpleLock { + lock(id: string): Promise + unlock(id: string): Promise +} + +//# resource = true +/** @internal */ +export declare class Website { + readonly url: string + constructor(props: { path: string, domain?: HostedZone }) +} + +//# resource = true +/** @internal */ +export declare class Schedule { + public constructor(expression: string, fn: () => Promise | void) +} diff --git a/src/runtime/srl/index.ts b/src/runtime/srl/index.ts new file mode 100644 index 0000000..3c6b545 --- /dev/null +++ b/src/runtime/srl/index.ts @@ -0,0 +1,14 @@ +//# moduleId = synapse:srl + +export * as net from './net' +export * as compute from './compute' +export * as storage from './storage' + +//# resource = true +export declare class Provider { + constructor(props?: any) +} + +// Re-exporting common resources so they're easier to find +// export { Function, HttpService } from './compute' +// export { Bucket, Queue, Table, Counter } from './storage' \ No newline at end of file diff --git a/src/runtime/srl/net/index.ts b/src/runtime/srl/net/index.ts new file mode 100644 index 0000000..b9ec7be --- /dev/null +++ b/src/runtime/srl/net/index.ts @@ -0,0 +1,107 @@ +//# moduleId = synapse:srl/net + +//--------------- NETWORKKING ---------------// + +interface RouteProps { + readonly type?: 'ipv4' | 'ipv6' + readonly destination: string + // LoadBalancer? +} + +// creates a route within a network +//# resource = true +/** @internal */ +export declare class Route { + constructor(network: RouteTable | Network, props: RouteProps) +} + +//# resource = true +/** @internal */ +export declare class RouteTable {} + +interface SubnetProps { + readonly cidrBlock?: string + readonly ipv6CidrBlock?: string +} + +//# resource = true +/** @internal */ +export declare class Subnet { + constructor(network: Network, props?: SubnetProps) +} + +//# resource = true +/** @internal */ +export declare class Network { + readonly subnets: Subnet[] +} + +//# resource = true +/** @internal */ +export declare class InternetGateway { + constructor() +} + +// For east-west traffic (IGW is north-south) +//# resource = true +/** @internal */ +export declare class TransitGateway { + constructor() + + public addNetwork(network: Network): void +} + +// for AWS these are always 'allow' +/** @internal */ +export interface NetworkRuleProps { + readonly type: 'ingress' | 'egress' + readonly priority?: number + readonly protocol: 'icmp' | 'tcp' | 'udp' | number // IPv4 protocol id + // TODO: icmp has extra settings for AWS + readonly port: number | [number, number] // only applicable to L4 protocols + // source or destination + readonly target: string // can be _a lot_ of different things +} + +//# resource = true +/** @internal */ +export declare class NetworkRule { + constructor(network: Network, props: NetworkRuleProps) +} + +//# resource = true +/** @internal */ +export declare class Firewall {} + +//# resource = true +export declare class HostedZone { + readonly name: string + constructor(name: string) + createSubdomain(name: string): HostedZone + + //# resource = true + createRecord(record: ResourceRecord): void +} + +interface NsRecord { + type: 'NS' + name: string + ttl: number + value: string +} + +interface AliasRecord { + type: 'A' + name: string + ttl: number + value: string +} + +interface CNameRecord { + type: 'CNAME' + name: string + ttl: number + value: string +} + +export type ResourceRecord = CNameRecord | NsRecord | AliasRecord diff --git a/src/runtime/srl/storage/index.ts b/src/runtime/srl/storage/index.ts new file mode 100644 index 0000000..4b509ac --- /dev/null +++ b/src/runtime/srl/storage/index.ts @@ -0,0 +1,141 @@ +//# moduleId = synapse:srl/storage + +import { HttpHandler } from 'synapse:http' +import { HostedZone } from 'synapse:srl/net' + +// FIXME: this only works if the declaration files are transformed to use the same +// ambient module specifier. A reference directive then needs to be added to the importing file + +// import { Bucket } from './bucket' +// export * from './bucket' + +//# resource = true +export declare class Queue { + send(val: T): Promise + /** @internal */ + consume(fn: (val: T) => U | Promise): Promise + + //# resource = true + on(eventType: 'message', cb: (message: T) => Promise | void): unknown +} + +//# resource = true +export declare class Table { + constructor() + get(key: K): Promise + /** @internal */ + getBatch(keys: K[]): Promise<{ key: K, value: V }[]> + set(key: K, val: V): Promise + /** @internal */ + setBatch(items: { key: K, value: V }[]): Promise + delete(key: K): Promise + /** @internal */ + values(): AsyncIterable +} + +//# resource = true +/** @internal */ +export declare class Secret { + constructor(envVar?: string) + get(): Promise +} + +// For CDN +/** @internal */ +export interface OriginOptions { + origin: string + targetPath: string + originPath?: string + allowedMethods: string[] +} + +//# resource = true +/** @internal */ +export declare class CDN { + readonly url: string + constructor(store: { + readonly bucket: Bucket + readonly indexKey?: string + readonly domain?: HostedZone + readonly additionalRoutes?: OriginOptions[] + readonly middleware?: HttpHandler + readonly compress?: boolean + }) + + //# resource = true + addOrigin(origin: OriginOptions): void +} + +//# resource = true +/** @internal */ +export declare class StaticDeployment { + constructor(store: Bucket, source?: string) + add(file: string): string +} + +//# resource = true +export declare class Counter { + constructor(init?: number) + get(): Promise + set(amount: number): Promise + inc(amount?: number): Promise +} + +//# resource = true +/** @internal */ +export declare class KeyedCounter { + constructor(init?: number) + get(key: string): Promise + set(key: string, amount: number): Promise + inc(key: string, amount?: number): Promise + dec(key: string, amount?: number): Promise +} + +//# resource = true +/** @internal */ +export declare class TTLCache { + /** TTL is in seconds */ + constructor(ttl?: number) + get(key: K): Promise + put(key: K, value: V): Promise + keys(): Promise + delete(key: K): Promise +} + +// import * as assert from 'node:assert' +// import { describe, it } from 'synapse:test' +import { DataPointer } from 'synapse:core' + +export type Encoding = 'utf-8' | 'base64' | 'base64url' | 'hex' + +//# resource = true +export declare class Bucket { + get(key: string): Promise + get(key: string, encoding: Encoding): Promise + put(key: string, blob: string | Uint8Array): Promise + list(prefix?: string): Promise + /** @internal */ + stat(key: string): Promise<{ size: number; contentType?: string }> + delete(key: string): Promise + + //# resource = true + /** @internal */ + addBlob(sourceLocation: string | DataPointer, key?: string, contentType?: string): string +} + +// describe('Bucket', () => { +// const bucket = new Bucket() + +// it('can have stuff', async () => { +// await bucket.put('foo', 'bar') +// const data = await bucket.get('foo', 'utf-8') +// assert.strictEqual(data, 'bar') +// }) +// }) + +//# resource = true +/** @internal */ +export declare class Stream implements AsyncIterable { + put(...values: T[]): Promise + [Symbol.asyncIterator](): AsyncIterator +} diff --git a/src/runtime/srl/websites.ts b/src/runtime/srl/websites.ts new file mode 100644 index 0000000..3a5d8f3 --- /dev/null +++ b/src/runtime/srl/websites.ts @@ -0,0 +1,58 @@ +///# moduleId = synapse:srl/websites + +import { HttpHandler, HttpFetcher, CapturedPattern } from 'synapse:http' +import { HostedZone } from 'synapse:srl/net' + + +export interface JSXElement< + P = any, + T extends string | JSXElementConstructor

= string | JSXElementConstructor

+> { + type: T + props: P + key: string | null +} + +export type JSXNode

= JSXElement

| Iterable> + +type JSXElementConstructor

= ((props: P) => JSXNode

) | (new (props: P) => Component) + +interface Component { + render(): JSXElement +} + +type FunctionComponent

= (props: P, context?: C) => U +// type ComponentType

= (new (props: P) => Component) | FunctionComponent

+type ComponentType

= FunctionComponent + +export interface Layout { + readonly parent?: Layout + // readonly stylesheet?: Stylesheet + readonly component: ComponentType<{ children: JSXNode }> +} + +export interface Page = {}, U = any> { + readonly layout: Layout + readonly component: ComponentType +} + +export interface RouteablePage = {}> extends Page { + readonly route: string +} + +//# resource = true +export declare class Website { + readonly url: string + constructor(options?: { domain?: HostedZone }) + + addAsset(source: string, name?: string, contentType?: string): string + + addPage(route: T, page: Page>): RouteablePage> + addPage(route: T, page: Page, U>, context: U): RouteablePage> + + // XXX: having the `string` overload makes things work correctly ??? + addHandler(route: T, handler: HttpHandler): HttpFetcher + addHandler(route: T, handler: HttpHandler): HttpFetcher + + bind(handler: (...args: T) => Promise | U): (...args: T) => Promise +} diff --git a/src/runtime/utils.ts b/src/runtime/utils.ts new file mode 100644 index 0000000..5790304 --- /dev/null +++ b/src/runtime/utils.ts @@ -0,0 +1,68 @@ +import * as vm from 'node:vm' +import * as path from 'node:path' +import { SyncFs } from '../system' +import { throwIfNotFileNotFoundError } from '../utils' + +export interface Context { + readonly vm: vm.Context + readonly globals: typeof globalThis +} + +export function copyGlobalThis(): Context { + const ctx = vm.createContext() + const globals = vm.runInContext('this', ctx) + const keys = new Set(Object.getOwnPropertyNames(globals)) + + const propDenyList = new Set([ + 'crypto', // Node adds a prop that throws if in a different context + 'console', // We want to add our own + ]) + + const descriptors = Object.entries(Object.getOwnPropertyDescriptors(globalThis)).filter(([k]) => !keys.has(k) && !propDenyList.has(k)) + + for (const [k, v] of descriptors) { + Object.defineProperty(globals, k, v) + } + + globals.ArrayBuffer = ArrayBuffer + // Needed for `esbuild` + globals.Uint8Array = Uint8Array + + globals.console = globalThis.console + + Object.defineProperty(globals, 'crypto', { + value: globalThis.crypto, + writable: false, + configurable: false, + }) + + return { vm: ctx, globals } +} + +export type CodeCache = ReturnType +export function createCodeCache(fs: Pick, cacheDir: string) { + function getCachedData(key: string): Buffer | undefined { + const filePath = path.resolve(cacheDir, key) + + try { + const d = fs.readFileSync(filePath) + + return Buffer.isBuffer(d) ? d : Buffer.from(d) + } catch (e) { + throwIfNotFileNotFoundError(e) + } + } + + function setCachedData(key: string, data: Uint8Array) { + const filePath = path.resolve(cacheDir, key) + fs.writeFileSync(filePath, data) + } + + function evictCachedData(key: string) { + const filePath = path.resolve(cacheDir, key) + fs.deleteFileSync(filePath) + } + + return { getCachedData, setCachedData, evictCachedData } +} + diff --git a/src/services/analytics/backend.ts b/src/services/analytics/backend.ts new file mode 100644 index 0000000..db5174c --- /dev/null +++ b/src/services/analytics/backend.ts @@ -0,0 +1,250 @@ +import * as storage from 'synapse:srl/storage' +import { Service, Client } from 'synapse:services' +import { RandomString } from 'synapse:key-service' +import { HttpError } from 'synapse:http' +import { randomUUID } from 'node:crypto' +import { getDeviceId } from './deviceId' +import { describe, it, expect } from 'synapse:test' + +export interface AnalyticsEvent { + readonly type: string + readonly timestamp: string // ISO + readonly attributes?: Record +} + +interface StoredAnalyticsEvent extends AnalyticsEvent { + readonly deviceId: string +} + +interface PostEventsRequest { + readonly batch: AnalyticsEvent[] +} + +interface DeviceIdentity { + readonly id: string +} + +function getEventKey(ev: Pick) { + const d = new Date(ev.timestamp) + + return `${d.getUTCFullYear()}/${(d.getUTCMonth() + 1).toString().padStart(2, '0')}/${(d.getUTCDate()).toString().padStart(2, '0')}` +} + +const maxBufferSize = 10 * 1024 * 1024 +const maxBufferDurationMs = 60_000 + +function createBufferedBucket2(bucket: storage.Bucket) { + interface BufferedData { + size: number + timer: NodeJS.Timer + chunks: Uint8Array[] + } + + const buffers: Record = {} + function getBuffer(key: string) { + if (key in buffers) { + return buffers[key] + } + + const timer = setTimeout(() => flush(key), maxBufferDurationMs) + + return buffers[key] = { + size: 0, + timer, + chunks: [], + } + } + + async function _flush(key: string, data: BufferedData) { + clearTimeout(+data.timer) + await bucket.put(key, Buffer.concat(data.chunks)) + } + + function flush(key: string) { + const data = buffers[key] + if (!data) { + return + // throw new Error('Missing buffered data') + } + + delete buffers[key] + + return _flush(key, data) + } + + function put(key: string, data: Uint8Array) { + const b = getBuffer(key) + b.size += data.byteLength + b.chunks.push(data) + + if (b.size >= maxBufferSize) { + return flush(key) + } + } + + return { put } +} + +function createBufferedBucket(bucket: storage.Bucket) { + interface BufferedData { + id: string + size: number + chunks: Uint8Array[] + } + + const buffers: Record = {} + function getBuffer(key: string) { + if (key in buffers) { + return buffers[key] + } + + return buffers[key] = { + id: randomUUID(), + size: 0, + chunks: [], + } + } + + const pendingResolves: Record void, (err: Error) => void][]> = {} + + async function _write(key: string, d: BufferedData) { + const k = `${key}/${d.id}` + + while (true) { + const pending = pendingResolves[k] + if (!pending) { + break + } + + delete pendingResolves[k] + + try { + await bucket.put(k, Buffer.concat(d.chunks)) + for (const r of pending) { + r[0]() + } + } catch (e) { + for (const r of pending) { + r[1](e as Error) + } + } + } + } + + const pendingWrites: Record> = {} + function write(key: string, d: BufferedData) { + const k = `${key}/${d.id}` + const p = new Promise((resolve, reject) => { + const arr = pendingResolves[k] ??= [] + arr.push([resolve, reject]) + }) + + if (k in pendingWrites) { + return p + } + + pendingWrites[k] = _write(key, d).finally(() => delete pendingWrites[k]) + + return p + } + + function put(key: string, data: Uint8Array) { + const b = getBuffer(key) + b.size += data.byteLength + b.chunks.push(data) + + if (b.size >= maxBufferSize) { + delete buffers[key] + } + + return write(key, b) + } + + return { put } +} + +interface GetEventsRequest { + from?: string + to?: string + type?: string +} + +class Analytics extends Service { + private readonly bucket = new storage.Bucket() + private readonly buffered = createBufferedBucket(this.bucket) + + public async postEvents(req: PostEventsRequest) { + const events = req.batch.map(ev => ({ ...ev, deviceId: this.context.id } satisfies StoredAnalyticsEvent)) + const promises: Promise[] = [] + for (const ev of events) { + const d = Buffer.from(JSON.stringify(ev) + '\n', 'utf-8') + promises.push(this.buffered.put(getEventKey(ev), d)) + } + + await Promise.all(promises) + } + + public async getEvents(req: GetEventsRequest) { + const t = req.from ? new Date(req.from) : new Date() + const k = getEventKey({ timestamp: t.toISOString() }) + const keys = await this.bucket.list(k) + const events: StoredAnalyticsEvent[] = [] + for (const k of keys) { + const d = await this.bucket.get(k, 'utf-8') + for (const l of d.split('\n')) { + if (l) { + events.push(JSON.parse(l)) + } + } + } + + return { events } + } +} + +// This token is embedded into the CLI as simple mechanism to prevent abuse +const secret = new RandomString(16).value + +const analytics = new Analytics() +analytics.addAuthorizer(async (authz, req) => { + const [scheme, rest] = authz.split(' ') + if (scheme !== 'Basic') { + throw new HttpError(`Bad scheme: ${scheme}`, { statusCode: 403 }) + } + + const [id, token] = Buffer.from(rest, 'base64').toString('utf-8').split(':') + if (token !== secret) { + throw new HttpError(`Invalid token`, { statusCode: 403 }) + } + + return { id } +}) + +// This isn't intended to be super secure. +analytics.setAuthorization(async () => { + const deviceId = await getDeviceId() + const authorization = `Basic ${Buffer.from(`${deviceId}:${secret}`).toString('base64')}` + + return authorization +}) + +export const client = analytics as Client + + +describe('analytics', () => { + function makeTestEvent(attr?: Record, time = new Date()): AnalyticsEvent { + return { + type: 'test', + timestamp: time.toISOString(), + attributes: attr, + } + } + + it('can send events', async () => { + const req: PostEventsRequest = { batch: [makeTestEvent(), makeTestEvent()] } + await client.postEvents(req) + const data = await analytics.getEvents({}) + expect(data.events.find(ev => ev.timestamp === req.batch[0].timestamp)) + }) +}) + diff --git a/src/services/analytics/daemon.ts b/src/services/analytics/daemon.ts new file mode 100644 index 0000000..5e92da0 --- /dev/null +++ b/src/services/analytics/daemon.ts @@ -0,0 +1,377 @@ +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import * as net from 'node:net' +import * as child_process from 'node:child_process' +import { Bundle } from 'synapse:lib' +import { logToFile } from '../../cli/logger' +import { getLogger } from '../../logging' +import { client, AnalyticsEvent } from './backend' +import { getLogsDirectory, getSocketsDirectory } from '../../workspaces' +import { ensureDir } from '../../system' + +const getSocketPath = () => path.resolve(getSocketsDirectory(), 'analytics.sock') + +interface IpcEventMessage { + readonly type: 'event' + readonly data: any +} + +interface IpcStdoutMessage { + readonly type: 'stdout' + readonly data: string | Buffer +} + +interface IpcErrorMessage { + readonly type: 'error' + readonly data: any +} + +type IpcMessage = IpcEventMessage | IpcStdoutMessage | IpcErrorMessage + +interface IpcCommand { + name: string + args: string[] +} + +interface AnalyticsCommand { + name: 'analytics' + args: AnalyticsEvent[] +} + +const hasClient = (function () { + try { + client.postEvents + return true + } catch { + return false + } +})() + +const getLogFile = () => path.resolve(getLogsDirectory(), 'analytics.log') +const getStartupLogs = () => path.resolve(getLogsDirectory(), 'analytics-startup.log') + +export async function startServer() { + const disposable = logToFile(getLogger(), undefined, getLogFile()) + const server = net.createServer() + + const inactivityTimer = setInterval(async () => { + if (sockets.size > 0) { + return + } + + getLogger().log('Shutting down due to inactivity') + await shutdown() + }, 300_000).unref() + + getLogger().log('Opening socket') + + const socketPath = getSocketPath() + + await new Promise((resolve, reject) => { + server.once('error', async e => { + if ((e as any).code === 'EACCES') { + await fs.mkdir(path.dirname(socketPath), { recursive: true }) + server.once('error', reject) + return server.listen(socketPath, resolve) + } + + if ((e as any).code !== 'EADDRINUSE') { + return reject(e) + } + + getLogger().log('Removing old socket') + await fs.rm(socketPath) + server.once('error', reject) + server.listen(socketPath, resolve) + }) + + server.listen(socketPath, resolve) + }) + + let timer: NodeJS.Timeout | undefined + const buffer: AnalyticsEvent[] = [] + function enqueue(...events: AnalyticsEvent[]) { + buffer.push(...events) + + if (timer !== undefined) { + clearTimeout(+timer) + } + + if (buffer.length >= 25) { + flush() + } else { + timer = setTimeout(flush, 30_000) + } + } + + async function _flush() { + while (buffer.length > 0) { + const batch = buffer.splice(0, 25) + await client.postEvents({ batch }) + } + } + + let p: Promise | undefined + function flush() { + return p ??= _flush().finally(() => (p = undefined)) + } + + async function shutdown() { + [...sockets].forEach(s => s.end()) + await new Promise((resolve, reject) => { + server.close(err => err ? reject(err) : resolve()) + }) + // TODO: we should use a lock when we start shutting down + // There's a race condition atm + await fs.rm(getLogFile()).catch(() => {}) + clearTimeout(timer) + await flush() + await disposable.dispose() + process.exit(0) + } + + async function handleRequest(command: IpcCommand | AnalyticsCommand) { + try { + if (command.name === 'shutdown') { + return await shutdown() + } + + return { + type: 'stdout', + data: '', + } + } catch (e) { + return { + type: 'error', + data: { + ...(e as any), + name: (e as any).name, + message: (e as any).message, + stack: (e as any).stack, + } + } + } + } + + const sockets = new Set() + + function sendMessage(socket: net.Socket, message: string | Buffer) { + return new Promise((resolve, reject) => { + socket.write(message, err => err ? reject(err) : resolve()) + }) + } + + async function broadcast(msg: string | Buffer) { + await Promise.all([...sockets].map(s => sendMessage(s, msg))) + } + + async function broadcastEvent(ev: any) { + await broadcast(JSON.stringify({ type: 'event', data: ev } satisfies IpcEventMessage) + '\n') + } + + + server.on('connection', socket => { + inactivityTimer.refresh().unref() + sockets.add(socket) + + socket.on('end', () => { + sockets.delete(socket) + }) + + socket.on('data', d => { + inactivityTimer.refresh().unref() + const cmd = JSON.parse(d.toString('utf-8')) as IpcCommand + if (cmd.name === 'analytics') { + const batch = (cmd as any).args as AnalyticsEvent[] + if (hasClient) { + enqueue(...batch) + } + + return + } + + handleRequest(cmd).then(resp => { + socket.write(JSON.stringify(resp) + '\n') + + }) + }) + + // socket.on('error', ...) + }) + + // server.on('close', async () => { + // await shutdown() + // }) + + getLogger().log('Sending ready message') + + process.send?.({ status: 'ready' }) +} + +async function startDaemon(daemonModule: string) { + const logFile = getStartupLogs() + await ensureDir(logFile) + const log = await fs.open(logFile, 'w') + const proc = child_process.fork( + daemonModule, + [], + { + stdio: ['ignore', log.fd, log.fd, 'ipc'], + detached: true, + } + ) + + await new Promise((resolve, reject) => { + function onMessage(ev: child_process.Serializable) { + if (typeof ev === 'object' && !!ev && 'status' in ev && ev.status === 'ready') { + close() + } + } + + function onExit(code: number | null, signal: NodeJS.Signals | null) { + if (code) { + close(new Error(`Non-zero exit code: ${code}\n logs: ${logFile}`)) + } else if (signal) { + close(new Error(`Received signal to exit: ${signal}`)) + } + close(new Error(`Process exited without sending a message`)) + } + + function close(err?: any) { + if (err) { + reject(err) + } else { + resolve() + } + proc.removeListener('message', onMessage) + proc.removeListener('error', close) + proc.removeListener('error', onExit) + } + + proc.on('message', onMessage) + proc.on('error', close) + proc.on('exit', onExit) + }).finally(() => log.close()) + + proc.unref() + proc.disconnect() +} + +async function startDaemonIfDown(daemonModule: string, socketPath = getSocketPath()) { + try { + return await openSocket(socketPath) + } catch(e) { + await startDaemon(daemonModule) + + return openSocket(socketPath) + } +} + +async function openSocket(socketPath: string) { + const socket = await new Promise((resolve, reject) => { + function complete(err?: Error) { + s.removeListener('error', complete) + s.removeListener('ready', complete) + + if (err) { + reject(err) + } else { + resolve(s) + } + } + + const s = net.connect(socketPath) + s.once('ready', complete) + s.once('error', complete) + }) + + return socket +} + +const startFn = new Bundle(startServer, { + immediatelyInvoke: true, +}) + +export async function connect() { + const socket = await startDaemonIfDown(startFn.destination) + socket.on('error', err => { + getLogger().error('analytics socket', err) + }) + + async function runCommand(name: string, args: string[] | AnalyticsEvent[]) { + const p = new Promise((resolve, reject) => { + function complete(err?: Error) { + socket.removeListener('exit', exit) + socket.removeListener('data', handleData) + + if (err) { + reject(err) + } else { + resolve() + } + } + + let buffer = '' + async function handleData(d: Buffer) { + buffer += d.toString('utf-8') + + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + + for (const l of lines) { + const msg = JSON.parse(l) as IpcMessage + + if (msg.type === 'error') { + complete(msg.data) + } else if (msg.type === 'stdout') { + complete() + } else { + getLogger().emit(msg.data.type, msg.data) + } + } + } + + function exit() { + complete(new Error(`Socket closed unexpectedly`)) + } + + socket.on('exit', exit) + socket.on('close', exit) + socket.on('data', handleData) + }) + + await new Promise(async (resolve, reject) => { + socket.write( + JSON.stringify({ name, args: args as any } satisfies IpcCommand), + err => err ? reject(err) : resolve() + ) + }) + + return p + } + + async function shutdown() { + return runCommand('shutdown', []) + } + + function sendAnalytics(events: AnalyticsEvent[]) { + return new Promise(async (resolve, reject) => { + socket.write( + JSON.stringify({ name: 'analytics', args: events as any } satisfies IpcCommand), + err => err ? reject(err) : resolve() + ) + }) + } + + async function dispose() { + await new Promise((resolve, reject) => socket.end(resolve)) + } + + return { + shutdown, + dispose, + sendAnalytics, + + destroySocket: () => socket.destroy(), + } +} \ No newline at end of file diff --git a/src/services/analytics/deviceId.ts b/src/services/analytics/deviceId.ts new file mode 100644 index 0000000..d19a363 --- /dev/null +++ b/src/services/analytics/deviceId.ts @@ -0,0 +1,71 @@ +import * as path from 'node:path' +import { getFs } from '../../execution' +import { getHash } from '../../utils' +import { createMemento } from '../../utils/memento' +import { runCommand } from '../../utils/process' +import { randomUUID } from 'node:crypto' +import { getUserSynapseDirectory } from '../../workspaces' + + // https://github.com/denisbrodbeck/machineid + +async function getDarwinId() { + const res = await runCommand('/usr/sbin/ioreg', ['-rd1', '-c', 'IOPlatformExpertDevice']) + const m = res.match(/"IOPlatformUUID" = "([0-9-]+)"/) + if (!m) { + return + } + + return getHash(m[1]) +} + +async function getLinuxId() { + const d = await getFs().readFile('/var/lib/dbus/machine-id', 'utf-8').catch(e => { + return getFs().readFile('/etc/machine-id', 'utf-8').catch(e => {}) + }) + + if (!d) { + return + } + + return getHash(d.trim()) +} + +async function getMachineId() { + switch (process.platform) { + case 'darwin': + return getDarwinId() + case 'linux': + return getLinuxId() + } +} + +let deviceId: string +async function _getDeviceId() { + const memento = createMemento(getFs(), path.resolve(getUserSynapseDirectory(), 'memento')) + const deviceId = await memento.get('deviceId') + if (deviceId) { + return deviceId + } + + const machineId = await getMachineId() + if (machineId) { + await memento.set('deviceId', machineId) + + return machineId + } + + const newDeviceId = getHash(randomUUID()) + await memento.set('deviceId', newDeviceId) + + return newDeviceId +} + +export async function getDeviceId() { + return deviceId ??= await _getDeviceId() +} + +async function approximateProjectId() { + // 1. Get the current root dir + // 2. Remove prefixes e.g. home dir + // 3. Hash it +} \ No newline at end of file diff --git a/src/services/analytics/index.ts b/src/services/analytics/index.ts new file mode 100644 index 0000000..5a43e1d --- /dev/null +++ b/src/services/analytics/index.ts @@ -0,0 +1,107 @@ +import { getLogger } from '../../logging' +import { readKey } from '../../cli/config' +import { memoize } from '../../utils' +import { AnalyticsEvent } from './backend' +import { connect } from './daemon' + +const pendingEvents = new Set>() + +export function emitEvent(ev: AnalyticsEvent) { + if (isAnalyticsDisabledByEnv()) { + return + } + + const p = _emit(ev).finally(() => pendingEvents.delete(p)) + pendingEvents.add(p) +} + +export function eagerlyStartDaemon() { + getDaemon() +} + +async function _emit(ev: AnalyticsEvent) { + const daemon = await getDaemon() + + return daemon?.sendAnalytics([ev]) +} + +// Sync check is kept separate so we can skip creating promises +const isAnalyticsDisabledByEnv = memoize(() => { + if (process.env['DO_NOT_TRACK'] && process.env['DO_NOT_TRACK'] !== '0') { + return true + } + + if (process.env['SYNAPSE_NO_ANALYTICS'] && process.env['SYNAPSE_NO_ANALYTICS'] !== '0') { + return true + } +}) + +const isAnalyticsDisabled = memoize(async () => { + return isAnalyticsDisabledByEnv() || (await readKey('cli.analytics')) === false +}) + +const getDaemon = memoize(async () => { + if (await isAnalyticsDisabled()) { + return + } + + try { + return await connect() + } catch (e) { + if (!(e as any).message.includes('has not been deployed')) { + getLogger().error(e) + } + } +}) + +export async function shutdown() { + const start = performance.now() + + if (pendingEvents.size === 0) { + if (getDaemon.cached) { + (await getDaemon())?.dispose() + } + return + } + + const timer = setTimeout(async () => { + if (getDaemon.cached) { + getLogger().warn(`Forcibly destroying analytics socket`) + ;(await getDaemon())?.destroySocket() + } + }, 100).unref() + + await Promise.all(pendingEvents) + ;(await getDaemon())?.dispose() + clearTimeout(timer) + getLogger().debug(`analytics shut down time: ${Math.floor((performance.now() - start) * 100) / 100}ms`) +} + +interface CommandEventAttributes { + readonly name: string + readonly duration: number + readonly errorCode?: string + // cliVersion + // OS + // arch + // maybe shell +} + +interface CommandEvent extends AnalyticsEvent { + readonly type: 'command' + readonly attributes: CommandEventAttributes +} + +export function emitCommandEvent(attributes: CommandEventAttributes) { + emitEvent({ + type: 'command', + timestamp: new Date().toISOString(), + attributes, + } satisfies CommandEvent) +} + +const legalNotice = ` +The Synapse CLI collects anonymous usage data. You can opt-out by running the command \`syn config synapse.cli.analytics false\`. + +For more information on what is collected, see the following documentation: +` \ No newline at end of file diff --git a/src/services/secrets/index.ts b/src/services/secrets/index.ts new file mode 100644 index 0000000..22c94d3 --- /dev/null +++ b/src/services/secrets/index.ts @@ -0,0 +1,92 @@ +import * as path from 'node:path' +import { getGlobalCacheDirectory } from '../../workspaces' +import { secrets } from '@cohesible/resources' +import { getFs } from '../../execution' +import { getLogger } from '../../logging' +import { throwIfNotFileNotFoundError } from '../../utils' +import { getInmemSecretService } from './inmem' + + +const secretsCache = new Map() + +// XXX: WE SHOULDN'T DO THIS +const getSecretsCacheFile = () => path.resolve( + getGlobalCacheDirectory(), // TODO: this should be a per-project (or per-program) cache + 'secrets.json' +) + +async function writeSecretsCache() { + const fs = getFs() + const data = Object.fromEntries( + [...secretsCache.entries()].map(([k, v]) => [k, { ...v, expiration: v.expiration ?? (new Date(Date.now() + 15 * 60 * 1000)).toISOString()}]) + ) + await fs.writeFile(getSecretsCacheFile(), JSON.stringify(data)) +} + +async function readSecretsCache() { + const fs = getFs() + const data = await fs.readFile(getSecretsCacheFile(), 'utf-8').catch(throwIfNotFileNotFoundError) + + if (!data) { + return + } + + for (const [k, v] of Object.entries(JSON.parse(data))) { + secretsCache.set(k, v as secrets.Secret) + } +} + +const useInmem = false + +async function getSecretValue(secretType: string) { + if (useInmem) { + return getInmemSecretService().getSecret(secretType) + } + + const envVar = secretType.toUpperCase().replaceAll('-', '_') + if (process.env[envVar]) { + return { value: process.env[envVar]! } + } + + const resp = await secrets.client.listSecrets() + const filtered = resp.filter(s => s.secretType === secretType) + if (filtered.length === 0) { + throw new Error(`No secrets found matching type: ${secretType}`) + } + + return secrets.client.getSecretValue(filtered[0].id) +} + +export async function getSecret(type: string) { + await readSecretsCache() + if (secretsCache.has(type)) { + const secret = secretsCache.get(type)! + if (!secret.expiration || (new Date(secret.expiration).getTime() > Date.now())) { + return secret.value + } + + getLogger().log(`Removing expired secret from cache: ${type}`) + } + + getLogger().log(`Getting secret type: ${type}`) + const secret = await getSecretValue(type) + secretsCache.set(type, secret) + await writeSecretsCache() + + return secret.value +} + +async function getOrCreateSecret(secretType: string) { + const resp = await secrets.client.listSecrets() + const filtered = resp.filter(s => s.kind === secretType) + if (filtered.length > 0) { + return filtered[0] + } + + return await secrets.client.createSecret({ secretType }) +} + +export async function putSecret(secretType: string, value: string, expiresIn?: number) { + const secret = await getOrCreateSecret(secretType) + await secrets.client.putSecretValue(secret.id, value) +} diff --git a/src/services/secrets/inmem.ts b/src/services/secrets/inmem.ts new file mode 100644 index 0000000..89afd56 --- /dev/null +++ b/src/services/secrets/inmem.ts @@ -0,0 +1,36 @@ +import { ServiceProvider, getServiceRegistry } from '../../deploy/registry' +import type { SecretProviderProps } from '../../runtime/modules/core' +import { memoize } from '../../utils' + + +export function createInmemSecretService() { + const providers = new Map() + + async function getSecret(secretType: string) { + const provider = [...providers.values()].find(x => x.secretType === secretType) + if (!provider) { + throw new Error(`No secret provider found: ${secretType}`) + } + + return await provider.getSecret() + } + + function _getBinding(): ServiceProvider { + return { + kind: 'secret-provider', + load: (id, config) => void providers.set(id, config), + unload: (id) => void providers.delete(id), + } + } + + return { + getSecret, + _getBinding, + } +} + +export const getInmemSecretService = memoize(createInmemSecretService) + +// getServiceRegistry().registerServiceProvider( +// getInmemSecretService()._getBinding() +// ) \ No newline at end of file diff --git a/src/static-solver/compiler.ts b/src/static-solver/compiler.ts new file mode 100644 index 0000000..d3075cd --- /dev/null +++ b/src/static-solver/compiler.ts @@ -0,0 +1,2432 @@ +import ts, { factory, isCallExpression } from 'typescript' +import { SourceMapHost, createVariableStatement, emitChunk, extract, failOnNode, getNodeLocation, getNullTransformationContext, isNonNullable, printNodes } from './utils' +import { isAssignmentExpression, Symbol, Scope, createGraphOmitGlobal, getContainingScope, unwrapScope, getSubscopeDfs, getReferencesInScope, getRootSymbol, RootScope, createGraph, getSubscopeContaining, getImmediatelyCapturedSymbols, getRootAndSuccessorSymbol, printSymbol } from './scopes' +import { createObjectLiteral, createPropertyAssignment, createSymbolPropertyName, createSyntheticComment, hashNode, memoize, removeModifiers } from '../utils' +import { SourceMapV3 } from '../runtime/sourceMaps' +import { liftScope } from './scopes' +import { ResourceTypeChecker } from '../compiler/resourceGraph' +import { throwIfCancelled, CancelError } from '../execution' + +function isSuperCallExpressionStatement(node: ts.Statement) { + return ( + ts.isExpressionStatement(node) && + ts.isCallExpression(node.expression) && + node.expression.expression.kind === ts.SyntaxKind.SuperKeyword + ) +} + +function createMovablePropertyName(factory = ts.factory) { + return createSymbolPropertyName('__moveable__', factory) +} + +function createSerializationData( + targetModule: string, + captured: ts.Expression[], + factory = ts.factory, + moduleType: 'esm' | 'cjs' +) { + const filename = moduleType === 'cjs' + ? factory.createIdentifier('__filename') + : factory.createPropertyAccessExpression( + factory.createPropertyAccessExpression(factory.createIdentifier('import'), 'meta'), + 'filename' + ) + + const moduleExpression = factory.createCallExpression( + factory.createIdentifier('__getPointer'), + undefined, + [ + filename, + factory.createStringLiteral(targetModule) + ] + ) + + return createObjectLiteral({ + valueType: 'function', + module: moduleExpression, + captured, + }, factory) +} + +function createModuleFunction( + serializationData: ts.Expression, + factory = ts.factory, +) { + return factory.createArrowFunction( + undefined, + undefined, + [], + undefined, + undefined, + factory.createBlock([ + factory.createReturnStatement(serializationData) + ], true) + ) +} + +function addModuleSymbolToFunction( + node: ts.FunctionDeclaration, + serializationData: ts.Expression, + factory = ts.factory, +) { + if (!node.name) { + failOnNode('Expected name', node) + } + + const access = factory.createElementAccessExpression( + factory.createIdentifier(node.name.text), + createMovablePropertyName(factory) + ) + + return factory.createExpressionStatement( + factory.createAssignment( + access, + createModuleFunction(serializationData, factory) + ) + ) +} + +function addModuleSymbolToMethod( + node: ts.MethodDeclaration, + serializationData: ts.Expression, + factory = ts.factory, +) { + if (!node.name) { + failOnNode('Expected name', node) + } + + if (ts.isPrivateIdentifier(node.name)) { + failOnNode('Cannot serialize private methods', node.name) + } + + const className = ts.isClassLike(node.parent) ? node.parent.name : undefined + if (!className) { + failOnNode('Expected class name', node.parent) + } + + const classIdent = factory.createIdentifier(className.text) + const accessExp = ts.isIdentifier(node.name) + ? factory.createPropertyAccessExpression(classIdent, node.name) + : factory.createElementAccessExpression( + factory.createIdentifier(className.text), + ts.isComputedPropertyName(node.name) ? node.name.expression : node.name + ) + const access = factory.createElementAccessExpression( + accessExp, + createMovablePropertyName(factory) + ) + + return factory.createExpressionStatement( + factory.createAssignment( + access, + createModuleFunction(serializationData, factory) + ) + ) +} + +function createAssignExpression(target: ts.Expression, value: ts.Expression, factory = ts.factory) { + return factory.createCallExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier('Object'), + 'assign' + ), + undefined, + [target, value] + ) +} + +function addModuleSymbolToFunctionExpression( + node: ts.FunctionExpression | ts.ArrowFunction, + serializationData: ts.Expression, + factory = ts.factory, +) { + const fn = factory.createArrowFunction( + undefined, + undefined, + [], + undefined, + undefined, + factory.createParenthesizedExpression(serializationData) + ) + + return createAssignExpression(node, factory.createObjectLiteralExpression([ + factory.createPropertyAssignment( + factory.createComputedPropertyName( + createSymbolPropertyName('__moveable__', factory) + ), + fn + ) + ]), factory) +} + + +interface Scopes { + readonly calls: (ts.NewExpression | ts.CallExpression)[] // | TaggedTemplateExpression | Decorator | InstanceofExpression + readonly classes: (ts.ClassDeclaration | ts.ClassExpression | ts.ObjectLiteralExpression)[] + readonly methods: (ts.MethodDeclaration | ts.GetAccessorDeclaration | ts.SetAccessorDeclaration)[] + readonly functions: (ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction | ts.ConstructorDeclaration)[] + readonly assignments: (ts.VariableDeclaration | ts.PropertyAssignment | ts.PropertyDeclaration)[] + + readonly throws: ts.ThrowStatement[] +} + +function getScopeType(node: ts.Node): keyof Scopes | undefined { + switch (node.kind) { + case ts.SyntaxKind.NewExpression: + case ts.SyntaxKind.CallExpression: + return 'calls' + + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.ClassExpression: + case ts.SyntaxKind.ObjectLiteralExpression: + return 'classes' + + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.GetAccessor: + case ts.SyntaxKind.SetAccessor: + return 'methods' + + case ts.SyntaxKind.FunctionDeclaration: + case ts.SyntaxKind.FunctionExpression: + case ts.SyntaxKind.ArrowFunction: + case ts.SyntaxKind.Constructor: + return 'functions' + + case ts.SyntaxKind.Parameter: + case ts.SyntaxKind.VariableDeclaration: + case ts.SyntaxKind.PropertyDeclaration: + case ts.SyntaxKind.PropertyAssignment: + return 'assignments' + + case ts.SyntaxKind.ThrowStatement: + return 'throws' + } +} + +export type ScopeTracker = ReturnType +export function createScopeTracker() { + type ScopesWithKey = { [P in keyof Scopes]: [Scopes[P][number], keyof Scopes][] } + + const scopes: Scopes = { + calls: [], + classes: [], + methods: [], + functions: [], + assignments: [], + + throws: [], + } + + function getScope(key: T, pos = -1): Scopes[T][number] | undefined { + return scopes[key].at(pos) + } + + function getScopes(...keys: T): Readonly { + return Array.from(iterateScopes.apply(undefined, keys)) + } + + function* iterateScopes(...keys: T): Generator { + if (keys.length === 1) { + return yield* scopes[keys[0]] + } + + const s = new Map(keys.map(k => [k, -1])) + for (let i = keyScope.length - 1; i >= 0; i--) { + const k = keyScope[i] + const j = s.get(k) + if (!j) continue + + yield scopes[k].at(j)! + s.set(k, j - 1) + } + } + + const keyScope: (keyof Scopes)[] = [] + function enter(node: U, fn: (node: U) => T): T { + const type = getScopeType(node) + if (type === undefined) { + return fn(node) + } + + // keys.push(type) + const arr = scopes[type] as ts.Node[] + arr.push(node) + keyScope.push(type) + + const result = fn(node) + arr.pop() + keyScope.pop() + + return result + } + + function dump() { + const stack = getScopes(...(Object.keys(scopes) as any)) + const lines: string[] = [] + for (let i = 0; i < stack.length; i++) { + lines.push(`${getNodeLocation(stack[i])} [${keyScope[i]}]`) + } + return lines + } + + return { enter, getScope, getScopes, iterateScopes, dump } +} + +// function first(iter: Iterator): T | undefined { +// const n = iter.next() +// if (n.done) { +// return +// } + +// return n.value +// } + +// function find(iter: Iterator, fn: (val: T) => val is U): U | undefined { +// const n = iter.next() +// if (n.done) { +// return +// } + +// if (fn(n.value)) { +// return n.value +// } +// } + +// function find2(iter: Iterator, fn: (val: T) => boolean): T | undefined { +// const n = iter.next() +// if (n.done) { +// return +// } + +// if (fn(n.value)) { +// return n.value +// } +// } + +// function getInstantiationName2(node: ts.Node, tracker: ReturnType): string { +// function getName(node: ts.Node): string { +// if (ts.isIdentifier(node)) { +// return node.text +// } + +// if (node.kind === ts.SyntaxKind.ThisKeyword) { +// const p = find2( +// tracker.iterateScopes('classes', 'functions'), +// n => !(n.kind === ts.SyntaxKind.Constructor || n.kind === ts.SyntaxKind.ArrowFunction) +// ) + +// // const parent = ts.findAncestor(node, ts.isClassDeclaration) ?? ts.findAncestor(node, ts.isClassExpression) ?? ts.findAncestor(node, ts.isObjectLiteralExpression) ?? ts.findAncestor(node, ts.isFunctionExpression) ?? ts.findAncestor(node, ts.isFunctionDeclaration) +// if (!p) { +// tracker.dump() +// failOnNode(`Failed to get container declaration`, node) +// } + +// return getName(p) +// } + +// if (ts.isObjectLiteralElement(node)) { +// if (!node.name) { +// return '' +// } + +// return getName(node.name) +// } + +// if (ts.isVariableDeclaration(node) || ts.isPropertyDeclaration(node)) { +// // if (!ts.isIdentifier(node.name)) { +// // failOnNode(`Could not get name of node`, node) +// // } + +// return getName(node.name) +// } + +// if (ts.isGetAccessorDeclaration(node) || ts.isSetAccessorDeclaration(node)) { +// return getName(node.name) +// } + +// if (ts.isClassDeclaration(node) || ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) { +// if (!node.name || !ts.isIdentifier(node.name)) { +// failOnNode(`Could not get name of declaration node`, node) +// } + +// return getName(node.name) +// } + +// if (ts.isCallExpression(node) || ts.isNewExpression(node)) { +// const parts = splitExpression(node.expression) +// const names = parts.map(getName).filter(isNonNullable) + +// return names.join('--') +// } + +// if (ts.isConstructorDeclaration(node)) { +// const parent = find(tracker.iterateScopes('classes'), ts.isClassDeclaration) +// if (!parent) { +// failOnNode('No class declaration found for constructor', node) +// } + +// return getName(parent)! +// } + +// if (ts.isPropertyAccessExpression(node)) { +// return [getName(node.expression), getName(node.name)].join('--') +// } + +// if (ts.isExpressionWithTypeArguments(node)) { +// const parent = find(tracker.iterateScopes('classes'), ts.isClassDeclaration) +// if (!parent) { +// failOnNode('No class declaration found for extends clause', node) +// } + +// return getName(parent) +// } + +// if (ts.isFunctionExpression(node)) { +// if (node.name) { +// return getName(node.name) +// } + +// return '__anonymous' +// } + +// if (node.kind === ts.SyntaxKind.SuperKeyword) { +// const parent = find(tracker.iterateScopes('classes'), ts.isClassDeclaration) +// const superClass = parent ? getSuperClassExpressions(parent)?.pop() : undefined +// if (!superClass) { +// failOnNode('No class declaration found when using `super` keyword', node) +// } + +// return getName(superClass) +// } + +// return '' +// // failOnNode('No name', node) +// } + + +// return getName(node) +// } + + +// const bindFunctions = new Set(['bindModel', 'bindObjectModel', 'bindFunctionModel']) +// function isInBindFunction(node: ts.Node) { +// return !!scopeTracker.getScopes('calls').find(x => +// x.kind === ts.SyntaxKind.CallExpression && +// bindFunctions.has(getCoreImportName(graphCompiler, x.expression) ?? '') +// ) +// } + +// - jsx(type, props, key) +// - jsxs(type, props, key) +// - jsxDEV(type, props, key, __source, __self) + +const assetsName = '__synapse_assets__' + +function createAssetAccess(assetName: string, factory = ts.factory) { + return factory.createElementAccessExpression( + factory.createIdentifier(assetsName), + factory.createStringLiteral(assetName) + ) +} + +export function createImporterExpression(moduleType: 'cjs' | 'esm', factory = ts.factory) { + if (moduleType === 'cjs') { + return factory.createIdentifier('__filename') + } + + return factory.createPropertyAccessExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier('import'), + 'meta' + ), + 'filename' + ) +} + +function createAssetCallExpression(target: ts.Expression) { + return ts.factory.createCallExpression( + ts.factory.createIdentifier('__createAsset'), + [], + [target, ts.factory.createIdentifier('__filename')] + ) +} + +type AssetsMap = Map + +function renderAssets(assets: AssetsMap) { + const entries = [...assets].map(([k, v]) => [k, createAssetCallExpression(v.literal)] as const) + + return createObjectLiteral({ + ...Object.fromEntries(entries), + [assetsName]: true, + }) +} + +// TODO: support this: +// /** @jsxImportSource preact */ + +// Also this I guess: +// /** @jsx h */ +// /** @jsxFrag Fragment */ +// import { h, Fragment } from 'preact' + +function transformJsx( + node: ts.Node, + jsxImport: ts.Identifier, + assets: AssetsMap, + mode: 'runtime' | 'infra', + factory = ts.factory +) { + const jsxExp = factory.createPropertyAccessExpression(jsxImport, 'jsx') + const jsxsExp = factory.createPropertyAccessExpression(jsxImport, 'jsxs') + const fragmentExp = factory.createPropertyAccessExpression(jsxImport, 'Fragment') + const staticChildren = new WeakMap() + + const tagStack: (ts.StringLiteral | ts.Identifier)[] = [] + + function isInImgTag() { + return tagStack[tagStack.length - 1]?.text === 'img' + } + + function getType(node: ts.JsxTagNameExpression) { + if (ts.isIdentifier(node)) { + if (node.text.charAt(0).toUpperCase() === node.text.charAt(0)) { + return node + } else { + return factory.createStringLiteral(node.text) + } + } + + if (ts.isPropertyAccessExpression(node)) { + failOnNode('TODO', node) + + // const member = node.name.text + // if (!ts.isIdentifier(node.expression)) { + // failOnNode('Not implemented', node.expression) + // } + + // return factory.createStringLiteral(`${node.expression.text}.${member}`) + } + + failOnNode('Not implemented', node) + } + + function getProperty(node: ts.JsxAttributeLike): [string, ts.Expression] | ts.Expression { + if (ts.isJsxSpreadAttribute(node)) { + return node.expression + } + + if (!ts.isIdentifier(node.name)) { + failOnNode('Not implemented', node) + } + + if (!node.initializer) { + return [node.name.text, factory.createTrue()] + } + + if (node.name.text === 'src' && ts.isStringLiteral(node.initializer) && isInImgTag() && (node.initializer.text.startsWith('./') || node.initializer.text.startsWith('../'))) { + return [node.name.text, transformAsset(node.initializer)] + } + + if (node.name.text === 'style' && ts.isJsxExpression(node.initializer)) { + if (node.initializer.expression && ts.isObjectLiteralExpression(node.initializer.expression)) { + return [node.name.text, transformInlineStyle(node.initializer.expression)] + } + } + + return [node.name.text, visitAttributeValue(node.initializer)] + } + + function transformInlineStyle(node: ts.ObjectLiteralExpression) { + const props = node.properties.map(p => { + if (ts.isPropertyAssignment(p) && ts.isIdentifier(p.name) && p.name.text === 'background') { + if (ts.isStringLiteral(p.initializer)) { + const transformed = maybeTransformUrl(p.initializer) + if (transformed) { + return factory.updatePropertyAssignment(p, p.name, transformed) + } + } + } + + return p + }) + + return factory.updateObjectLiteralExpression(node, props) + } + + function maybeTransformUrl(node: ts.StringLiteral) { + const urlIndex = node.text.indexOf('url(') + if (urlIndex !== -1) { + const start = urlIndex + 5 + const endIndex = node.text.indexOf(')', start) + if (endIndex !== -1) { + const url = node.text.slice(start, endIndex - 1) + if (url.startsWith('./') || url.startsWith('../')) { + assets.set(url, { literal: factory.createStringLiteral(url) }) + + return factory.createTemplateExpression( + factory.createTemplateHead(node.text.slice(0, start)), + [ + factory.createTemplateSpan( + createAssetAccess(url, factory), + factory.createTemplateTail(node.text.slice(endIndex - 1)) + ) + ] + ) + } + } + } + } + + function transformAsset(node: ts.StringLiteral): ts.Expression { + const literal = node.text + assets.set(literal, { literal: node }) + + return createAssetAccess(literal, factory) + } + + function visitAttributes(node: ts.JsxAttributes) { + const props = node.properties.map(getProperty) + const keyIndex = props.findIndex(v => Array.isArray(v) && v[0] === 'key') + const key = keyIndex !== -1 ? (extract(props, keyIndex) as [string, ts.Expression])[1] : undefined + + const attributes = props.map(v => { + if (Array.isArray(v)) { + return createPropertyAssignment(factory, v[0], v[1]) + } + + return factory.createSpreadAssignment(v) + }) + + return { key, attributes } + } + + interface VNodeProps { + key?: ts.Expression + children?: ts.Expression + attributes?: ts.ObjectLiteralElementLike[] + } + + function createVNode(type: ts.Expression, props?: VNodeProps) { + const key = props?.key + const elements = props?.attributes ?? [] + if (props?.children) { + elements.push(createPropertyAssignment(factory, 'children', props.children)) + } + + const attrExp = factory.createObjectLiteralExpression(elements, true) + const isStaticChildren = props?.children && staticChildren.get(props?.children) + + return factory.createCallExpression( + isStaticChildren ? jsxsExp : jsxExp, + undefined, + !key ? [type, attrExp] : [type, attrExp, key] + ) + } + + function visitChildren(children: ts.NodeArray): ts.Expression | undefined { + const mapped = children + .filter(c => !(ts.isJsxText(c) && c.containsOnlyTriviaWhiteSpaces)) + .filter(c => !(ts.isJsxExpression(c) && !c.expression)) + .map(c => { + if (ts.isJsxText(c)) { + return factory.createStringLiteral(c.text) + } + + return visitAttributeValue(c) + }) + + if (mapped.length === 0) { + return + } + + if (mapped.length === 1) { + return visitExpression(mapped[0]) + } + + const result = factory.createArrayLiteralExpression(mapped.map(visitExpression), true) + staticChildren.set(result, true) + + return result + } + + function visitAttributeValue(node: ts.JsxAttributeValue): ts.Expression { + if (ts.isStringLiteral(node)) { + return node + } + + if (ts.isJsxExpression(node)) { + if (!node.expression) { + failOnNode('Empty expression', node) + } + + return visitExpression(node.expression) + } + + if (ts.isJsxFragment(node)) { + return visitJsxFragment(node) + } + + if (ts.isJsxSelfClosingElement(node)) { + return visitJsxSelfClosingElement(node) + } + + return visitJsxElement(node) + } + + function visitJsxFragment(node: ts.JsxFragment) { + const children = visitChildren(node.children) + + return createVNode(fragmentExp, children ? { children } : undefined) + } + + function visitJsxSelfClosingElement(node: ts.JsxSelfClosingElement): ts.Expression { + const type = getType(node.tagName) + tagStack.push(type) + const attributes = visitAttributes(node.attributes) + tagStack.pop() + + return createVNode(type, attributes) + } + + function visitJsxElement(node: ts.JsxElement): ts.Expression { + const type = getType(node.openingElement.tagName) + tagStack.push(type) + const attributes = visitAttributes(node.openingElement.attributes) + const children = visitChildren(node.children) + tagStack.pop() + + return createVNode(type, children ? { ...attributes, children } : attributes) + } + + function visitExpression(node: ts.Expression): ts.Expression { + return visit(node) as ts.Expression + } + + const context = getNullTransformationContext() + + function visit(node: ts.Node): ts.Node { + if (ts.isJsxElement(node)) { + return visitJsxElement(node) + } + if (ts.isJsxSelfClosingElement(node)) { + return visitJsxSelfClosingElement(node) + } + if (ts.isJsxFragment(node)) { + return visitJsxFragment(node) + } + if (ts.isJsxExpression(node)) { + if (!node.expression) { + failOnNode('Empty expression', node) + } + + return visit(node.expression) + } + + return ts.visitEachChild(node, visit, context) + } + + return visit(node) +} + +function createJsxRuntime(options: ts.CompilerOptions, factory = ts.factory) { + const importSource = options.jsxImportSource ?? 'react' + + function createImportDecl(spec: string) { + const ident = factory.createIdentifier('import_jsx_runtime') + const decl = factory.createImportDeclaration( + undefined, + factory.createImportClause( + false, + undefined, + factory.createNamespaceImport(ident) + ), + factory.createStringLiteral(spec) + ) + + return { ident, decl } + } + + switch (options.jsx) { + case ts.JsxEmit.ReactJSX: + return createImportDecl(`${importSource}/jsx-runtime`) + default: + throw new Error(`JSX emit kind not implemented: ${options.jsx}`) + } +} + +type ClauseReplacement = [clause: ts.HeritageClause, ident: ts.Identifier] | undefined + +function addDeserializeConstructor( + node: ts.ClassExpression | ts.ClassDeclaration, + clauseReplacement: ClauseReplacement, + factory: ts.NodeFactory +) { + const descIdent = factory.createIdentifier('desc') + const tag = createSymbolPropertyName('deserialize', factory) + const fields = node.members.filter(isPrivateField) + const privateFieldsIdent = factory.createIdentifier('__privateFields') + const privateFields = factory.createVariableStatement( + undefined, + factory.createVariableDeclarationList( + [factory.createVariableDeclaration( + privateFieldsIdent, + undefined, + undefined, + factory.createCallExpression( + factory.createPropertyAccessExpression( + factory.createPropertyAccessExpression(descIdent, '__privateFields'), + 'pop' + ), + undefined, + [] + ) + )], + ts.NodeFlags.Const + ) + ) + + const callSuper = factory.createExpressionStatement( + factory.createCallChain( + factory.createElementAccessExpression(factory.createSuper(), tag), + factory.createToken(ts.SyntaxKind.QuestionDotToken), + undefined, + [descIdent] + ) + ) + + const hasSuper = !!node.heritageClauses?.find(c => c.token === ts.SyntaxKind.ExtendsKeyword)?.types?.[0]?.expression + const preamble = hasSuper ? [callSuper, privateFields] : [privateFields] + + const method = factory.createMethodDeclaration( + undefined, + undefined, + factory.createComputedPropertyName(tag), + undefined, + undefined, + [factory.createParameterDeclaration(undefined, undefined, descIdent)], + undefined, + factory.createBlock([ + ...preamble, + ...fields.map(decl => factory.createExpressionStatement(factory.createAssignment( + factory.createPropertyAccessExpression( + factory.createThis(), + factory.createIdentifier(getMappedPrivateName(node, decl.name)), + ), + factory.createElementAccessExpression( + privateFieldsIdent, + factory.createStringLiteral(decl.name.text, true) + ) + ))) + ], true) + ) + + const transformed = transformPrivateMembers(node, getNullTransformationContext()) + const heritageClauses = !clauseReplacement ? undefined : node.heritageClauses?.map(c => { + if (ts.getOriginalNode(c) === ts.getOriginalNode(clauseReplacement[0])) { + const clauseExp = factory.createExpressionWithTypeArguments(clauseReplacement[1], []) + + return factory.updateHeritageClause(c, [clauseExp]) + } + + return c + }) + + const props: ClassProps = { + members: [...transformed.members, method], + heritageClauses: heritageClauses, + } + + return updateClass(transformed, props, factory) +} + +function isAssignedTo(node: ts.Node) { + if (ts.isBinaryExpression(node.parent)) { + return isAssignmentExpression(node.parent) && node.parent.left === node + } +} + +function getPrivateAccessExpressionSymbol(sym: Symbol): Symbol | undefined { + if (!sym.parent) { + return + } + + if (!sym.name.startsWith('#')) { + return getPrivateAccessExpressionSymbol(sym.parent) + } + + const ref = sym.references[0] + if (!ref || !ts.isPropertyAccessExpression(ref) || !ts.isPrivateIdentifier(ref.name)) { + return + } + + return sym +} + +interface SymbolMapping { + identifier: ts.Identifier + bound?: boolean + isDefault?: boolean + lateBound?: boolean + moduleSpec?: string + replacementStack?: ts.Expression[] +} + +const replacementStacks = new Map() + +function rewriteCapturedSymbols( + scope: Scope, + captured: Symbol[], + globals: Symbol[], + circularRefs: Set, + runtimeTransformer?: (node: ts.Node) => ts.Node, + infraTransformer?: (node: ts.Node, depth: number) => ts.Node, + depth = 0, + factory = ts.factory +) { + const refs = new Map( + [...captured, ...globals].map(c => [c, getReferencesInScope(c, scope)]), + ) + + // Any symbol that is reassigned cannot be directly replaced + // TODO: call expressions with a `this` target cannot be captured directly + // The binding should probably be applied to all symbols unless we know the original + // declaration was `const` + const reduced = new Map() + const boundSymbols = new Set() + const defaultImports = new Set() + for (const [sym, nodes] of refs.entries()) { + if (circularRefs.has(sym) && !sym.parent) { + boundSymbols.add(sym) + reduced.set(sym, nodes) + } else if (!sym.parent && sym.references.some(isAssignedTo)) { + boundSymbols.add(sym) + reduced.set(sym, nodes) + } else if (sym.parent && sym.parent.references.some(isAssignedTo)) { + // The first check is to handle this case: + // ```ts + // const c = [] + // function f(u, v) { + // const a = c[u.value.id] ??= [] + // a[v.value.id] = r + // } + // ``` + // + // Right now this logic will always choose to bind `c` + // + // The symbol `a[v.value.id]` should be ignored because it's local + // to the scope. The only symbol we care about is `c[u.value.id]` + // + // Because the argument is computed _and_ local to the scope, we + // cannot bind `c[u.value.id]` + + if (sym.parent.computed && sym.parent.parent) { + boundSymbols.add(sym.parent.parent) + reduced.set(sym.parent.parent, getReferencesInScope(sym.parent.parent, scope)) + } else { + boundSymbols.add(sym.parent) + reduced.set(sym.parent, getReferencesInScope(sym.parent, scope)) + } + } else if ( + sym.variableDeclaration && + !(sym.variableDeclaration.parent.flags & ts.NodeFlags.Const) && // XXX: handles `for (const ...)` decls + (!sym.variableDeclaration.initializer || (sym.variableDeclaration.parent.flags & ts.NodeFlags.Let) // Any `let` binding is considered potentially mutable + )) { + boundSymbols.add(sym) + reduced.set(sym, nodes) + } else { + const privateExp = getPrivateAccessExpressionSymbol(sym) + if (privateExp) { + reduced.set(privateExp, getReferencesInScope(privateExp, scope)) + + continue + } + + const root = getRootSymbol(sym) + + const importClause = root.importClause + if (!importClause) { + reduced.set(root, getReferencesInScope(root, scope)) + + continue + } + + // Don't split up the module if we reference it directly + if (refs.has(root)) { + reduced.set(root, getReferencesInScope(root, scope)) + const name = importClause.name + if (name?.text === sym.name) { + defaultImports.add(sym) + } + + continue + } + + const bindings = importClause.namedBindings + if (!bindings) { + reduced.set(root, getReferencesInScope(root, scope)) + + continue + } + + if (!ts.isNamespaceImport(bindings)) { + reduced.set(root, getReferencesInScope(root, scope)) + + continue + } + + // Symbol is from a module, let's reduce it + let didReduce = false + for (const [name, member] of root.members.entries()) { + const nodes = getReferencesInScope(member, scope) + if (nodes.length > 0) { + reduced.set(member, nodes) + didReduce = true + } + } + if (!didReduce) { + reduced.set(root, getReferencesInScope(root, scope)) + } + } + } + + // Map all symbols to a valid identifier. Root symbols can be used as-is + const idents = new Map() + for (const sym of reduced.keys()) { + if (!replacementStacks.has(sym)) { + replacementStacks.set(sym, []) + } + + const isBound = boundSymbols.has(sym) + const isThis = sym.name === 'this' + const text = (sym.parent || isBound) ? `__${idents.size}` : isThis ? '__thisArg' : sym.name + idents.set(sym, { + identifier: factory.createIdentifier(text), + bound: isBound, + isDefault: defaultImports.has(sym), + lateBound: circularRefs.has(sym), + // moduleSpec, + replacementStack: replacementStacks.get(sym), + }) + } + + const replacements = new Map() + for (const [sym, nodes] of reduced) { + const { identifier, bound } = idents.get(sym)! + const newNode = bound + ? factory.createPropertyAccessExpression(identifier, sym.name) + : identifier + + replacementStacks.get(sym)![depth] = newNode + + for (const n of nodes) { + const exp = ts.isShorthandPropertyAssignment(n.parent) + ? factory.createPropertyAssignment(n.parent.name, cloneNode(newNode) as ts.Expression) + : cloneNode(newNode) + + ts.setTextRange(exp, n) + ts.setSourceMapRange(exp, n) + + replacements.set(n, exp) + } + } + + // Replace all non-root references with the identifier + const context = getNullTransformationContext() + function visit(node: ts.Node): ts.Node { + if (replacements.has(node)) { + return replacements.get(node)! + } + + return ts.visitEachChild(node, visit, context) + } + + const inner = scope.node + + try { + return { + node: visit(runtimeTransformer?.(inner) ?? inner), + // TODO: this doesn't need to be recursive + infraNode: visit(infraTransformer?.(inner, depth + 1) ?? inner), // TODO: can be made more efficient. We are creating orphaned files atm + parameters: idents, + } + } catch (e) { + if (e instanceof CancelError) { + throw e + } + throw new Error(`Failed to rewrite symbols at: ${getNodeLocation(scope.node)}`, { cause: e }) + } +} + +const getCloneNode = memoize(function () { + if (!('cloneNode' in ts.factory)) { + throw new Error('"cloneNode" is missing from the typescript module') + } + + return ts.factory.cloneNode as (node: ts.Node) => ts.Node +}) + +function cloneNode(node: ts.Node): ts.Node { + return getCloneNode()(node) +} + + +// Perf notes +// +// Benchmarking was done using this function: +// ```ts +// function benchmark(fn: () => string) { +// let x: bigint = 0n +// const start = Date.now() +// for (let i = 0; i < 100_000_000; i++) { +// x += BigInt(fn().length) +// } +// +// return Date.now() - start +// } +// +// For the following cases: +// a - val => val where val = 'foo' (baseline) +// b - val => val[0] where val = ['foo'] +// c - val => val.c where val = { c: 'foo' } +// +// Functions were constructed as closures e.g. `const y = (b => () => b[0]); const b = x(['foo'])` +// +// Results: +// * SpiderMonkey +// a - 2874ms +// b - 2963ms +// c - 2950ms +// * V8 +// a - 436ms +// b - 194ms (!!!) +// c - 438ms +// +// Conclusion: using arrays for bindings is preferred on V8 + +function getBindingName(target: ts.Expression) { + if (!ts.isPropertyAccessExpression(target)) { + failOnNode('Expected a property access expression', target) + } + + // if (!ts.isIdentifier(target.name)) { + // failOnNode('Not an identifier', target.name) + // } + + return target.name +} + +// This impl. doesn't work correctly in "scoped" cases. It's only kept around because +// it handles capturing symbols that don't have a corresponding declaration/statement +function renderLegacyBoundSymbol(symbol: Symbol, target: ts.Expression, lateBound = false, factory = ts.factory) { + const assignment = ts.isIdentifier(target) + ? factory.createShorthandPropertyAssignment(target) + : factory.createPropertyAssignment(getBindingName(target), target) + + return factory.createObjectLiteralExpression([ + assignment, + factory.createPropertyAssignment( + factory.createComputedPropertyName(createSymbolPropertyName('symbolId', factory)), + createObjectLiteral({ + id: symbol.id, + origin: factory.createIdentifier('__filename'), + lateBound, + }, factory) + ) + ]) +} + +function renderBoundSymbol(symbol: Symbol, target: ts.Expression, lateBound = false, factory = ts.factory) { + if (symbol.name.includes('.')) { + const message = `Unexpected symbol "${symbol.name}" [computed: ${symbol.computed}]` + const decl = symbol.declaration ?? symbol.parent?.declaration + if (decl) { + failOnNode(message, decl) + } + throw new Error(message) + } + + const assignment = ts.isIdentifier(target) + ? factory.createShorthandPropertyAssignment(target) + : factory.createPropertyAssignment(getBindingName(target), target) + + const elements: ts.ObjectLiteralElementLike[] = [assignment] + if (lateBound) { + elements.push(factory.createPropertyAssignment( + factory.createComputedPropertyName(createSymbolPropertyName('symbolId', factory)), + createObjectLiteral({ + id: symbol.id, // This id is local to the _module_ !!! + origin: factory.createIdentifier('__filename'), + lateBound, + }, factory) + )) + } + + const objExp = factory.createObjectLiteralExpression(elements) + + const symName = `__symbol${symbol.id}` + const tmpDecl = factory.createVariableStatement( + undefined, + factory.createVariableDeclarationList([ + factory.createVariableDeclaration(symName) + ], ts.NodeFlags.Let) + ) + + const lazyFn = factory.createArrowFunction(undefined, undefined, [], undefined, undefined, factory.createBinaryExpression( + factory.createIdentifier(symName), + ts.SyntaxKind.QuestionQuestionEqualsToken, + objExp + )) + + const fnName = `__getSymbol${symbol.id}` + const lazyFnDecl = createVariableStatement(fnName, lazyFn) + + return { + statements: [tmpDecl, lazyFnDecl], + expression: factory.createCallExpression(factory.createIdentifier(fnName), undefined, []), + } +} + +function getMappedSymbolExpression( + symbol: Symbol, + replacementStack: ts.Expression[] = [], + depth = 0 +) { + if (replacementStack && depth > 0) { + // It's probably ok to just check for `mapping.replacementStack[depth - 1]` + for (let i = depth - 1; i >= 0; i--) { + if (replacementStack[i]) { + if (i !== depth - 1) { + throw new Error(`Nope: ${i} !== ${depth - 1}`) + } + return replacementStack[i] + } + } + + // return replacementStack[depth - 1] + } + + if (!symbol.parent) { + return factory.createIdentifier(symbol.name) + } + + return cloneNode(symbol.references[0]!) as ts.Expression +} + +function renderConstSymbol(symbol: Symbol, mapping: Omit, depth = 0, printDebug = false) { + const exp = getMappedSymbolExpression(symbol, mapping.replacementStack, depth) + if (printDebug) { + const rootSym = getRootSymbol(symbol) + const location = rootSym.declaration ? getNodeLocation(rootSym.declaration) : undefined + if (location) { + ts.setSyntheticLeadingComments(exp, [createSyntheticComment(` ${location}`)]) + } + } + + return exp +} + +function createExportedDefaultFunction( + parameters: ts.ParameterDeclaration[], + block: ts.Block, + moduleType: 'cjs' | 'esm' = 'cjs' +) { + if (moduleType === 'esm') { + return factory.createFunctionDeclaration( + [ + factory.createModifier(ts.SyntaxKind.ExportKeyword), + factory.createModifier(ts.SyntaxKind.DefaultKeyword), + ], + undefined, + undefined, // factory.createIdentifier(name), + undefined, + parameters, + undefined, + block + ) + } + + return factory.createExportAssignment(undefined, true, factory.createFunctionExpression( + [], + undefined, + undefined, // factory.createIdentifier(name), + undefined, + parameters, + undefined, + block + )) +} + +export interface CompiledFile { + readonly sourceNode: ts.Node + readonly artifactName: string + readonly name: string + readonly source: string + readonly path: string + readonly data: string + readonly infraData: string // Used only when consuming a module in + readonly parameters: [Symbol, SymbolMapping][] + readonly assets?: AssetsMap + readonly sourcesmaps?: { + readonly runtime: SourceMapV3 + readonly infra: SourceMapV3 + } + // These are the names of all artifacts referenced by the source + readonly artifactDependencies: string[] +} + +// const cache = new Map() +// const nodeIds = new Map() +// const getNodeId = (node: ts.Node) => { +// if (nodeIds.has(node)) { +// return nodeIds.get(node)! +// } + +// const id = nodeIds.size +// nodeIds.set(node, id) + +// return id +// } + +// const getNodeCacheKey = (node: ts.Node, depth: number) => { +// const id = getNodeId(ts.getOriginalNode(node)) + +// return `${id}-${depth}` +// } + +function isClassElementModifier(modifier: ts.ModifierLike) { + switch (modifier.kind) { + case ts.SyntaxKind.StaticKeyword: + case ts.SyntaxKind.PublicKeyword: + case ts.SyntaxKind.PrivateKeyword: + case ts.SyntaxKind.ProtectedKeyword: + return true + } + + return false +} + +function convertMethodToFunction(node: ts.MethodDeclaration) { + const name = ts.factory.createIdentifier('__fn') + const modifiers = node.modifiers?.filter(x => !isClassElementModifier(x)) + + return ts.factory.createFunctionDeclaration( + modifiers, + node.asteriskToken, + name, + undefined, + node.parameters, + undefined, + node.body + ) +} + +// DUPLICATED IN `resourceGraph.ts` +function getImportName(sym: Symbol, clause: ts.ImportClause) { + const parent = sym.declaration?.parent + if (parent === clause) { + return 'default' + } + + if (parent && ts.isImportSpecifier(parent)) { + const name = parent.propertyName ?? parent.name + + return name.text + } +} + +// DUPLICATED IN `resourceGraph.ts` +function resolveModuleSpecifier(node: ts.Node) { + const moduleSpec = (node as ts.StringLiteral).text + + return { + specifier: moduleSpec, + } +} + +// DUPLICATED IN `resourceGraph.ts` +function getNameComponents(sym: Symbol): { specifier?: string; name?: string } { + if (sym.parent) { + const parentFqn = getNameComponents(sym.parent) + + return { + ...parentFqn, + name: parentFqn.name ? `${parentFqn.name}.${sym.name}` : sym.name, + } + } + + if (sym.importClause) { + const { specifier } = resolveModuleSpecifier(sym.importClause.parent.moduleSpecifier) + const name = getImportName(sym, sym.importClause) + + return { specifier, name } + } + + return { name: sym.name } +} + +export function getModuleType(opt: ts.ModuleKind | undefined): 'cjs' | 'esm' { + switch (opt) { + case ts.ModuleKind.ES2015: + case ts.ModuleKind.ES2020: + case ts.ModuleKind.ES2022: + case ts.ModuleKind.ESNext: + return 'esm' + + case undefined: + case ts.ModuleKind.Node16: + case ts.ModuleKind.NodeNext: + case ts.ModuleKind.CommonJS: + return 'cjs' + + default: + throw new Error(`Module kind not supported: ${opt}`) // FIXME: make this user friendly + } +} + +export function createGraphCompiler( + sourceMapHost: SourceMapHost, + compilerOptions: ts.CompilerOptions, + moduleType = getModuleType(compilerOptions.module) +) { + const rootGraphs = new Map() + const compiled = new Map>() + + const dependencyStack: Set[] = [] + + function getGraph(node: ts.SourceFile) { + if (rootGraphs.has(node)) { + return rootGraphs.get(node)! + } + + const graph = createGraph(node) + rootGraphs.set(node, graph) + + return graph + } + + function getSymbol(node: ts.Node) { + const graph = getGraph(node.getSourceFile()) + + return graph.symbols.get(node) + } + + function isDeclared(node: ts.Node) { + return !!getSymbol(ts.getOriginalNode(node))?.isDeclared + } + + function getJsxRuntime() { + return createJsxRuntime(compilerOptions) + } + + const capturedSymbols = new Map() + function getCaptured(sym: Symbol): Symbol[] { + if (capturedSymbols.has(sym)) { + return capturedSymbols.get(sym)! + } + + const scope = getContainingScope(sym) + if (!scope.node) { + const r: Symbol[] = [] + capturedSymbols.set(sym, r) + + return r + } + + const captured = getImmediatelyCapturedSymbols(scope) + capturedSymbols.set(sym, captured) + + return captured + } + + const dependencies = new Map>() + function getAllDependencies(sym: Symbol): Set { + if (dependencies.has(sym)) { + return dependencies.get(sym)! + } + + const deps = new Set() + dependencies.set(sym, deps) + for (const s of getCaptured(sym)) { + deps.add(s) + getAllDependencies(s).forEach(c => deps.add(c)) + } + + return deps + } + + function getCaptured2(node: ts.Node) { + const sourceFile = ts.getOriginalNode(node).getSourceFile() + const graph = getSubscopeContaining(getGraph(sourceFile), sourceFile) + const targetGraph = getSubscopeDfs(graph, node) + if (!targetGraph) { + failOnNode('No graph found', node) + } + if (targetGraph === graph) { + failOnNode('Got source file graph', node) + } + + const { captured } = liftScope(targetGraph) + + return captured + } + + function isCircularReference(currentSymbol: Symbol, nextSymbol: Symbol) { + return getAllDependencies(nextSymbol).has(currentSymbol) + } + + function liftNode( + node: ts.Node, + factory: ts.NodeFactory, + runtimeTransformer?: (node: ts.Node) => ts.Node, + infraTransformer?: (node: ts.Node, depth: number) => ts.Node, + clauseReplacement: ClauseReplacement = undefined, + jsxRuntime?: ts.Identifier, + excluded: ts.Node[] = [], + depth = 0 + ) { + const sourceFile = ts.getOriginalNode(node).getSourceFile() + const graph = getSubscopeContaining(getGraph(sourceFile), sourceFile) + //const immediateEnclosingScope = ts.findAncestor(node, n => (ts.isSourceFile(n) || ts.isFunctionDeclaration(n)) && n !== node) + //const parentGraph = immediateEnclosingScope === sourceFile ? graph : getSubgraphContaining(graph, immediateEnclosingScope!) + const targetGraph = getSubscopeDfs(graph, node) + if (!targetGraph) { + failOnNode('No graph found', node) + } + if (targetGraph === graph) { + failOnNode('Got source file graph', node) + } + + const { globals, captured } = liftScope( + targetGraph, + [], // ['console'] + excluded.map(n => ts.getOriginalNode(n)).map(n => getSubscopeDfs(graph, n)!) + ) + + // process.stdout.write(`${globals.length}, ${captured.length}\n`) + const extracted: ts.Node[] = [] + + const circularRefs = !targetGraph.symbol + ? new Set() + : new Set(captured.filter(s => isCircularReference(targetGraph.symbol!, s))) + + const rewritten = rewriteCapturedSymbols( + targetGraph, + captured, + globals, + circularRefs, + runtimeTransformer, + infraTransformer, + depth, + factory + ) + + if (clauseReplacement) { + rewritten.parameters.set({ name: clauseReplacement[1].text } as any, { identifier: clauseReplacement[1] }) + } + + if (jsxRuntime) { + rewritten.parameters.set({ name: jsxRuntime.text } as any, { identifier: jsxRuntime }) + } + + // Symbols are sorted by the # of parents first followed by their + // symbol id as a proxy for their position + // + // TODO: rename all identifiers to use their symbol id + function compareSymbols(a: Symbol, b: Symbol): number { + if (!a.parent && !b.parent) { + return a.id - b.id + } else if (a.parent && !b.parent) { + return 1 + } else if (!a.parent && b.parent) { + return -1 + } + + return compareSymbols(a.parent!, b.parent!) || (a.id - b.id) + } + + // The order of the parameters matters (obviously...) so we convert the map to + // an array to ensure that the parameters are always read in the same order + const parameters = [...rewritten.parameters.entries()].sort((a, b) => compareSymbols(a[0], b[0])) + + function finalize(node: ts.Node) { + if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { + return addDeserializeConstructor(node, clauseReplacement, factory) + } + + if (ts.isMethodDeclaration(node)) { + return convertMethodToFunction(node) + } + + return ts.isVariableDeclaration(node) ? node.initializer! : node + } + + const assets: AssetsMap = new Map() + + function createClosure(body: ts.Node, mode: 'runtime' | 'infra') { + const finalized = finalize(jsxRuntime ? transformJsx(body, jsxRuntime, assets, mode, factory) : body) + const withoutModifiers = (ts.isFunctionDeclaration(finalized) || ts.isClassDeclaration(finalized)) + ? removeModifiers(finalized, [ts.SyntaxKind.ExportKeyword, ts.SyntaxKind.DefaultKeyword], factory) + : factory.createReturnStatement(finalized as ts.Expression) + + ts.setSourceMapRange(withoutModifiers, targetGraph!.node) + + const statements = ts.isFunctionDeclaration(finalized) || ts.isClassDeclaration(finalized) + ? [ + withoutModifiers, + factory.createReturnStatement(finalized.name) // TODO: handle `export default class/function` + ] + : [withoutModifiers] + + const block = factory.createBlock(statements, true) + // note: values that are exclusively used during instantiation do not need to be captured for + // serialization/deserialization of class instances + const params = parameters.map(c => factory.createParameterDeclaration(undefined, undefined, c[1].identifier)) + if (assets.size > 0) { + params.push(factory.createParameterDeclaration(undefined, undefined, factory.createIdentifier(assetsName))) + } + + return [ + ...extracted, + createExportedDefaultFunction(params, block, moduleType) + ] + } + + return { + extracted: createClosure(rewritten.node, 'runtime'), + extractedInfra: createClosure(rewritten.infraNode, 'infra'), + parameters, + assets: assets.size > 0 ? assets : undefined, + } + } + + const consumers: ((file: CompiledFile) => Promise | void)[] = [] + function onEmitFile(consumer: (file: CompiledFile) => Promise | void) { + consumers.push(consumer) + } + + function emitFile(file: CompiledFile) { + for (const consumer of consumers) { + consumer(file) + } + } + + function compileNode( + name: string, + node: ts.Node, + factory: ts.NodeFactory, + runtimeTransformer?: (node: ts.Node) => ts.Node, + infraTransformer?: (node: ts.Node, depth: number) => ts.Node, + clauseReplacement: ClauseReplacement = undefined, + jsxRuntime?: ts.Identifier, + excluded: ts.Node[] = [], + depth = 0 + ) { + const sourceFile = node.getSourceFile() + if (!sourceFile) { + failOnNode('Missing source file', node) + } + + if (!compiled.has(sourceFile.fileName)) { + compiled.set(sourceFile.fileName, new Map()) + } + + const chunks = compiled.get(sourceFile.fileName)! + if (!chunks.has(name)) { + // ~4s for `doCompile` + // `doCompile` pops the stack + dependencyStack.push(new Set()) + + const chunk = doCompile() + chunks.set(name, chunk) + emitFile(chunk) + } + + const res = chunks.get(name)! + const artifactName = res.artifactName + if (dependencyStack.length > 0) { + dependencyStack[dependencyStack.length - 1].add(artifactName) + } + + return { + depth, + captured: res.parameters, + artifactName, + assets: res.assets, + } + + function doCompile() { + // `liftNode` takes ~3300ms total (`solver.ts` is almost 500ms??) + const { extracted, extractedInfra, parameters, assets } = liftNode(node, factory, runtimeTransformer, infraTransformer, clauseReplacement, jsxRuntime, excluded, depth) + const outfile = sourceFile.fileName.replace(/\.(t|j)(sx?)$/, `-${name}.$1$2`) + // `emitChunk` takes ~1700ms total for both calls + const emitSourceMap = !!compilerOptions.sourceMap + const result = emitChunk(sourceMapHost, sourceFile, extracted as ts.Statement[], { emitSourceMap }) + const resultInfra = emitChunk(sourceMapHost, sourceFile, extractedInfra as ts.Statement[], { emitSourceMap }) + + return { + sourceNode: ts.getOriginalNode(node), + name, + source: sourceFile.fileName, + path: outfile, + data: result.text, + infraData: resultInfra.text, + parameters, + assets, + artifactName: name, + artifactDependencies: Array.from(dependencyStack[dependencyStack.length - 1]), + sourcesmaps: emitSourceMap ? { + runtime: result.sourcemap!, + infra: resultInfra.sourcemap!, + } : undefined, + } + } + } + + return { getSymbol, getJsxRuntime, liftNode, compileNode, compiled, onEmitFile, isDeclared, getAllDependencies, getCaptured2, moduleType } +} + +interface StatementUpdate { + readonly before?: ts.Statement[] + readonly after?: ts.Statement[] +} + +function* updateStatements( + statements: ts.Statement[] | ts.NodeArray, + updates: Map +) { + for (const node of statements) { + const updateToApply = updates.get(ts.getOriginalNode(node) as ts.Statement) + const before = updateToApply?.reduce((a, b) => a.concat(b.before ?? []), [] as ts.Statement[]) + const after = updateToApply?.reduce((a, b) => a.concat(b.after ?? []), [] as ts.Statement[]) + + if (before) { + yield* before + } + + yield node + + if (after) { + yield* after + } + } +} + +function getFirstStatementOfParent(child: ts.Node) { + const parent = child.parent + if ((ts.isSourceFile(parent) || ts.isBlock(parent)) && parent.statements.length > 0) { + return parent.statements[0] + } + + failOnNode('Parent does not have any statements', child) +} + +function getAnonymousFunctionName(node: ts.ArrowFunction | ts.FunctionExpression) { + if (ts.isVariableDeclaration(node.parent) && node.parent.initializer === node && ts.isIdentifier(node.parent.name)) { + const name = node.parent.name.text + if (node.parent.parent.flags & ts.NodeFlags.Const) { // Is this even correct???? + return name + } + + return `${name}_${hashNode(node).slice(0, 16)}` + } + + if (ts.isPropertyAssignment(node.parent) && node.parent.initializer === node && ts.isIdentifier(node.parent.name)) { + const name = node.parent.name.text + + return `${name}_${hashNode(node).slice(0, 16)}` + } + + return `function_${hashNode(node).slice(0, 16)}` +} + +export function createRuntimeTransformer( + compiler: ReturnType, + resourceTypeChecker?: ResourceTypeChecker +): (node: ts.Node) => ts.Node { + const context = getNullTransformationContext() + + function visitCallExpression(node: ts.CallExpression) { + const sym = node.expression.getSourceFile() ? compiler.getSymbol(node.expression) : undefined + if (!sym) { + return ts.visitEachChild(node, visit, context) + } + + const callableMember = resourceTypeChecker?.getCallableMemberName(sym) + if (!callableMember) { + return ts.visitEachChild(node, visit, context) + } + + return factory.updateCallExpression( + node, + factory.createPropertyAccessExpression(node.expression, callableMember), + node.typeArguments, + node.arguments.map(visit) as ts.Expression[], + ) + } + + function visit(node: ts.Node): ts.Node { + if (ts.isCallExpression(node)) { + return visitCallExpression(node) + } + + if (ts.isImportDeclaration(node)) { + const spec = (node.moduleSpecifier as ts.StringLiteral).text + if (spec.endsWith('.zig')) { + return ts.factory.updateImportDeclaration( + node, + node.modifiers, + node.importClause, + ts.factory.createStringLiteral(spec.replace(/\.zig$/, '.zig.js')), + undefined, + ) + } + } + + return ts.visitEachChild(node, visit, context) + } + + return visit +} + +export function createSerializer( + compiler: ReturnType, + resourceTypeChecker?: ResourceTypeChecker +) { + const names = new Set() + const nameMap = new Map() + function getUniqueName(node: ts.Node, name: string) { + if (nameMap.has(node)) { + return nameMap.get(node)! + } + + let count = 0 + const getName = () => count === 0 ? name : `${name}_${count}` + while (names.has(getName())) count++ + + const result = getName() + names.add(result) + nameMap.set(node, result) + + return result + } + + // This function will re-compile any emitted files to include serialization data + function createInfraTransformer(name: string, innerTransformer?: (node: ts.Node) => ts.Node): (node: ts.Node, depth: number) => ts.Node { + const context = getNullTransformationContext() + + return (node, depth) => { + throwIfCancelled() + + const isJsx = !!ts.getOriginalNode(node).getSourceFile().fileName.match(/\.(t|j)sx$/) + const jsxRuntime = isJsx ? compiler.getJsxRuntime() : undefined + const transformer = createTransformer(context, innerTransformer, name, jsxRuntime?.ident, depth) + + // First transform adds the '__moveable__' symbol + // Second transform deals with `__scope__` + + const withMoveable = ts.visitEachChild(node, transformer.visit, context) + + // `innerTransformer` is 346ms + return innerTransformer?.(withMoveable) ?? withMoveable + } + } + + const runtimeTransformer = createRuntimeTransformer(compiler, resourceTypeChecker) + + function createTransformer( + context = getNullTransformationContext(), + innerTransformer?: (node: ts.Node) => ts.Node, + namePrefix?: string, + jsxRuntime?: ts.Identifier, + depth = 0 + ) { + // Only set for jsx currently + let markedForUseServer: Set | undefined + + // statements to add _after_ the target node + const updates = new Map() + function addStatementUpdate(node: ts.Statement, update: StatementUpdate) { + if (!updates.has(node)) { + updates.set(node, []) + } + updates.get(node)!.push(update) + } + + // Why is this prefixed `hoist` when it doesn't hoist anything? + function hoistSerializationData(node: ts.FunctionDeclaration, name: string, captured: ts.Expression[]) { + const serializationData = createSerializationData(name, captured, context.factory, compiler.moduleType) + addStatementUpdate(ts.getOriginalNode(node) as ts.Statement, { + after: [addModuleSymbolToFunction(node, serializationData, context.factory)] + }) + } + + function hoistMethodSerializationData(node: ts.MethodDeclaration, name: string, captured: ts.Expression[]) { + const serializationData = createSerializationData(name, captured, context.factory, compiler.moduleType) + addStatementUpdate(ts.getOriginalNode(node).parent as ts.Statement, { + after: [addModuleSymbolToMethod(node, serializationData, context.factory)] + }) + } + + function addUseServerSymbol(node: ts.FunctionDeclaration | ts.VariableDeclaration) { + const sym = createSymbolPropertyName('synapse.useServer', context.factory) + + function createAssignment(ident: ts.Identifier) { + return context.factory.createExpressionStatement( + context.factory.createAssignment( + context.factory.createElementAccessExpression(ident, sym), + context.factory.createTrue(), + ) + ) + } + + if (node.kind === ts.SyntaxKind.VariableDeclaration) { + const statement = ts.getOriginalNode(node as ts.VariableDeclaration).parent.parent as ts.Statement + addStatementUpdate(statement, { + after: [createAssignment((node as ts.VariableDeclaration).name as ts.Identifier)] + }) + } else { + const statement = ts.getOriginalNode(node) as ts.Statement + addStatementUpdate(statement, { + after: [createAssignment((node as ts.FunctionDeclaration).name as ts.Identifier)] + }) + } + } + + const boundSymbolExpressions = new Map() + function renderSymbol(symbol: Symbol, mapping: SymbolMapping, depth: number) { + if (!mapping.bound) { + return renderConstSymbol(symbol, mapping, depth) + } + + if (boundSymbolExpressions.has(symbol)) { + return boundSymbolExpressions.get(symbol)! + } + + const exp = getMappedSymbolExpression(symbol, mapping.replacementStack, depth) + const transforms = renderBoundSymbol(symbol, exp, mapping.lateBound) + if (!symbol.declaration) { + const fallback = renderLegacyBoundSymbol(symbol, exp, mapping.lateBound) + boundSymbolExpressions.set(symbol, fallback) + //throw new Error(`Missing symbol declaration: ${symbol.name}`) + return fallback + } + + const statement = ts.isVariableDeclaration(symbol.declaration) + ? symbol.declaration.parent.parent + : symbol.declaration + + if (!ts.isStatement(statement)) { + const fallback = renderLegacyBoundSymbol(symbol, exp, mapping.lateBound) + boundSymbolExpressions.set(symbol, fallback) + // failOnNode('Not a statement', statement) + return fallback + } + + boundSymbolExpressions.set(symbol, transforms.expression) + + addStatementUpdate(ts.getOriginalNode(statement) as ts.Statement, { + after: transforms.statements, + }) + + return transforms.expression + } + + function renderCapturedSymbols(mappings: [Symbol, SymbolMapping][], depth = 0, assets?: AssetsMap) { + const base = mappings.map(([x, v]) => renderSymbol(x, v, depth)) + if (assets) { + base.push(renderAssets(assets)) + } + + return base + } + + // FIXME: if an external class is used for deserialization and is also embedded into a module export + // then `instanceof` won't work between "moved" instances and instantiations within the export + // + // Isolating every declaration is one way to solve this + function extractClassDeclaration(node: ts.ClassDeclaration) { + node = ts.getOriginalNode(node) as ts.ClassDeclaration + const name = getName(node) + + // XXX: visit heritage clauses first + nameStack.push(name) + node.heritageClauses?.forEach(visit) + nameStack.pop() + + const clauseReplacement = Array.from(mappedClauses.entries()) + .map(([k, v]) => [k, v.ident] as NonNullable) + .find(c => node.heritageClauses?.includes(ts.getOriginalNode(c[0]) as any)) + + const excluded = clauseReplacement ? [clauseReplacement[0]] : undefined + + return { + ...compiler.compileNode(name, node, context.factory, runtimeTransformer, createInfraTransformer(name, innerTransformer), clauseReplacement, jsxRuntime, excluded, depth), + clauseReplacement, + } + } + + const nameStack: string[] = namePrefix ? [namePrefix] : [] + function getName(node: ts.Node) { + if (node.kind === ts.SyntaxKind.SuperKeyword) { + return nameStack.length === 0 ? 'super' : nameStack[nameStack.length - 1] + '_' + 'super' + } + + const original = ts.getOriginalNode(node) as ts.ClassDeclaration | ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction + const name = original.name?.text ?? getAnonymousFunctionName(node as any) + + if (nameStack.length === 0) { + return getUniqueName(original, name) + } + + return getUniqueName(original, nameStack[nameStack.length - 1] + '_' + name) + } + + const visited = new Map() + function visit(node: ts.Node): ts.Node { + const key = ts.getOriginalNode(node) + if (visited.has(key)) { + return visited.get(key)! + } + + const result = transform() + visited.set(key, result) + + return result + + function transform() { + if (ts.isClassDeclaration(node)) { + return visitClassDeclaration(node) + } + + if (ts.isHeritageClause(node)) { + return visitHeritageClause(node) + } + + if (ts.isFunctionDeclaration(node)) { + return visitFunctionDeclaration(node) + } + + // if (ts.isMethodDeclaration(node)) { + // return visitMethodDeclaration(node) + // } + + if (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) { + return visitArrowFunctionOrExpression(node) + } + + if (ts.isBlock(node)) { + return visitBlock(node) + } + + if (ts.isSourceFile(node)) { + return visitSourceFile(node) + } + + // The type is implied + if (markedForUseServer?.has(node)) { + addUseServerSymbol(node as ts.VariableDeclaration) + } + + return ts.visitEachChild(node, visit, context) + } + } + + function visitClassDeclaration(node: ts.ClassDeclaration) { + if (compiler.isDeclared(node)) { + return node + } + + const r = extractClassDeclaration(node) + const name = getName(node) + nameStack.push(name) + const visitedClass = ts.visitEachChild(node, visit, context) + nameStack.pop() + + return addSerializerSymbolToClass( + visitedClass, + createSerializationData( + r.artifactName, + renderCapturedSymbols(r.captured, r.depth, r.assets), + factory, + compiler.moduleType, + ), + r.clauseReplacement, + context, + ) + } + + function visitMethodDeclaration(node: ts.MethodDeclaration) { + const name = getName(node) + const r = compiler.compileNode(name, node, context.factory, runtimeTransformer, createInfraTransformer(name, innerTransformer), undefined, jsxRuntime, undefined, depth) + hoistMethodSerializationData(node, r.artifactName, renderCapturedSymbols(r.captured, r.depth, r.assets)) + + nameStack.push(name) + const res = ts.visitEachChild(node, visit, context) + nameStack.pop() + + return res + } + + function visitArrowFunctionOrExpression(node: ts.ArrowFunction | ts.FunctionExpression) { + // if (!compiler.canSerialize(node)) { + // return node + // } + + const name = getName(node) + const r = compiler.compileNode(name, node, context.factory, runtimeTransformer, createInfraTransformer(name, innerTransformer), undefined, jsxRuntime, undefined, depth) + + nameStack.push(name) + const visitedFn = ts.visitEachChild(node, visit, context) + nameStack.pop() + + return addModuleSymbolToFunctionExpression( + visitedFn, + createSerializationData( + r.artifactName, + renderCapturedSymbols(r.captured, r.depth, r.assets), + factory, + compiler.moduleType, + ), + context.factory, + ) + } + + function visitFunctionDeclaration(node: ts.FunctionDeclaration) { + // Overload + if (!node.body) { + return node + } + + if (markedForUseServer?.has(node)) { + addUseServerSymbol(node) + } + + const name = getName(node) + const r = compiler.compileNode(name, node, context.factory, runtimeTransformer, createInfraTransformer(name, innerTransformer), undefined, jsxRuntime, undefined, depth) + hoistSerializationData(node, r.artifactName, renderCapturedSymbols(r.captured, r.depth, r.assets)) + + nameStack.push(name) + const res = ts.visitEachChild(node, visit, context) + nameStack.pop() + + return res + } + + function visitBlock(node: ts.Block) { + node = ts.visitEachChild(node, visit, context) + + return context.factory.updateBlock( + node, + Array.from(updateStatements(node.statements, updates)), + ) + } + + const mappedClauses = new Map() + function visitHeritageClause(node: ts.HeritageClause) { + if (mappedClauses.has(node)) { + return mappedClauses.get(node)!.res + } + + if (node.token !== ts.SyntaxKind.ExtendsKeyword || !isCallExpression(getInnerExp(node.types[0]))) { + return ts.visitEachChild(node, visit, context) + } + + const name = getName(ts.factory.createSuper()) + const ident = ts.factory.createIdentifier(name) + const updatedExp = visit(node.types[0].expression) + if (!ts.isExpression(updatedExp)) { + failOnNode('Not an expression', updatedExp) + } + + const statement = ts.findAncestor(node, ts.isStatement) + if (!statement) { + failOnNode('Node is not apart of a statement', node) + } + + const decl = createVariableStatement(ident, updatedExp) + addStatementUpdate(statement, { before: [decl] }) + + const res = factory.updateHeritageClause( + node, + [factory.updateExpressionWithTypeArguments(node.types[0], ident, undefined)] + ) + + mappedClauses.set(node, { ident, res }) + + return res + } + + function visitSourceFile(node: ts.SourceFile) { + const isJsx = !!node.fileName.match(/\.(t|j)sx$/) + if (isJsx) { + const runtime = compiler.getJsxRuntime() + jsxRuntime = runtime.ident + if (node.statements.length > 0) { + addStatementUpdate(node.statements[0], { + before: [runtime.decl] + }) + } + markedForUseServer = resourceTypeChecker?.getMarkedNodes(node) + } + + node = ts.visitEachChild(node, visit, context) + const statements = Array.from(updateStatements(node.statements, updates)) + + return context.factory.updateSourceFile( + node, + statements, + node.isDeclarationFile, + node.referencedFiles, + node.typeReferenceDirectives, + node.hasNoDefaultLib, + node.libReferenceDirectives + ) + } + + return { visit } + } + + function transform(node: ts.Node) { + const result = ts.transform(node, [c => createTransformer(c).visit]) + + return printNodes(result.transformed) + } + + return { createTransformer, transform } +} + +function getInnerExp(node: ts.Expression): ts.Expression { + if (ts.isParenthesizedExpression(node) || ts.isAsExpression(node) || ts.isExpressionWithTypeArguments(node)) { + return getInnerExp(node.expression) + } + + return node +} + +function getMappedPrivateName( + node: ts.ClassDeclaration | ts.ClassExpression, + memberName: ts.PrivateIdentifier, + baseName = node.name ? `__${node.name.text}` : `__` // TODO: check super classes for private members +) { + return `${baseName}${memberName.text.replace(/^#/, '_')}` +} + +// Private methods will not work with `Reflect.construct` +function transformPrivateMembers( + node: ts.ClassDeclaration | ts.ClassExpression, + context: ts.TransformationContext +) { + const methods = node.members.filter(isPrivateMethod) + const fields = node.members.filter(isPrivateField) + + const mapped = new Map() + for (const m of [...methods, ...fields]) { + mapped.set(m.name.text, getMappedPrivateName(node, m.name)) + } + + function visit(node: ts.Node): ts.Node { + if (ts.isPrivateIdentifier(node)) { + const newName = mapped.get(node.text) + if (newName) { + return context.factory.createIdentifier(newName) + } + } + + return ts.visitEachChild(node, visit, context) + } + + + return ts.visitEachChild(node, visit, context) +} + +interface ClassProps { + members?: ts.ClassElement[] + heritageClauses?: ts.HeritageClause[] +} + +function updateClass(node: ts.ClassDeclaration | ts.ClassExpression, props: ClassProps, factory = ts.factory) { + if (ts.isClassDeclaration(node)) { + return factory.updateClassDeclaration( + node, + node.modifiers, + node.name, + node.typeParameters, + props.heritageClauses ?? node.heritageClauses, + props.members ?? node.members, + ) + } else { + return factory.updateClassExpression( + node, + node.modifiers, + node.name, + node.typeParameters, + props.heritageClauses ?? node.heritageClauses, + props.members ?? node.members, + ) + } +} + +function addSerializerSymbolToClass( + node: ts.ClassDeclaration | ts.ClassExpression, + serializationData: ts.Expression, + clauseReplacement: [clause: ts.HeritageClause, ident: ts.Identifier] | undefined, + context: ts.TransformationContext, +) { + const factory = context.factory + const serializeSymbol = createSymbolPropertyName('serialize') + const moveableSymbol = createSymbolPropertyName('__moveable__') + + const privateFields = Object.fromEntries(getPrivateFields(node).map(n => [ + (n.name! as ts.PrivateIdentifier).text, + factory.createPropertyAccessExpression( + factory.createThis(), + (n.name! as ts.PrivateIdentifier).text + ) + ])) + + const ident = factory.createIdentifier("__privateFields") + const init = factory.createBinaryExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier("desc"), + ident + ), + factory.createToken(ts.SyntaxKind.QuestionQuestionEqualsToken), + factory.createArrayLiteralExpression( + [], + false + ) + ) + const assignment = factory.createVariableStatement( + undefined, + factory.createVariableDeclarationList( + [factory.createVariableDeclaration( + ident, + undefined, + undefined, + init + )], + ts.NodeFlags.Const + ) + ) + + const push = factory.createCallExpression( + factory.createPropertyAccessExpression( + ident, + 'push' + ), + undefined, + [createObjectLiteral(privateFields, factory)] + ) + + const description = { + __privateFields: ident, + } + + // Private members will live on a stack + // Top of the stack will have private fields for the base class + // Each constructor pops the stack before returning + + const superClassExp = node.heritageClauses?.find(c => c.token === ts.SyntaxKind.ExtendsKeyword)?.types?.[0]?.expression + + const resultIdent = factory.createIdentifier('result') + const result = factory.createVariableStatement( + undefined, + factory.createVariableDeclarationList( + [factory.createVariableDeclaration( + resultIdent, + undefined, + undefined, + factory.createObjectLiteralExpression( + [ + ...createObjectLiteral(description, factory).properties, + factory.createSpreadAssignment( + factory.createIdentifier('desc') + ) + ], + true + ) + )], + ts.NodeFlags.Const + ) + ) + + + const serialize = factory.createMethodDeclaration( + // [factory.createToken(ts.SyntaxKind.StaticKeyword)], + undefined, + undefined, + factory.createComputedPropertyName(serializeSymbol), + undefined, + undefined, + [factory.createParameterDeclaration(undefined, undefined, 'desc', undefined, undefined, createObjectLiteral({}, factory))], + undefined, + factory.createBlock([ + assignment, + factory.createExpressionStatement(push), + result, + superClassExp !== undefined + ? factory.createReturnStatement( + factory.createBinaryExpression( + factory.createCallChain( + factory.createElementAccessExpression( + factory.createSuper(), + serializeSymbol + ), + factory.createToken(ts.SyntaxKind.QuestionDotToken), + undefined, + [resultIdent] + ), + factory.createToken(ts.SyntaxKind.QuestionQuestionToken), + resultIdent + ) + ) + : factory.createReturnStatement(resultIdent) + ], true) + ) + + + const move = factory.createMethodDeclaration( + [factory.createToken(ts.SyntaxKind.StaticKeyword)], + undefined, + factory.createComputedPropertyName(moveableSymbol), + undefined, + undefined, + [], + undefined, + factory.createBlock([ + factory.createReturnStatement(serializationData) + ], true) + ) + + const heritageClauses = !clauseReplacement ? undefined : node.heritageClauses?.map(c => { + if (ts.getOriginalNode(c) === ts.getOriginalNode(clauseReplacement[0])) { + const clauseExp = factory.createExpressionWithTypeArguments(clauseReplacement[1], []) + + return factory.updateHeritageClause(c, [clauseExp]) + } + + return c + }) + + const props: ClassProps = { + members: [...node.members, serialize, move], + heritageClauses: heritageClauses, + } + + return updateClass(node, props, factory) +} + +function getPrivateFields(node: ts.ClassDeclaration | ts.ClassExpression) { + return node.members.filter(isPrivateField) +} + +function isPrivateField(node: ts.ClassElement): node is ts.PropertyDeclaration & { name: ts.PrivateIdentifier } { + return !!node.name && ts.isPrivateIdentifier(node.name) && ts.isPropertyDeclaration(node) +} + +function isPrivateMethod(node: ts.ClassElement): node is ts.MethodDeclaration & { name: ts.PrivateIdentifier } { + return !!node.name && ts.isPrivateIdentifier(node.name) && ts.isMethodDeclaration(node) +} + + diff --git a/src/static-solver/index.ts b/src/static-solver/index.ts new file mode 100644 index 0000000..9a44bb3 --- /dev/null +++ b/src/static-solver/index.ts @@ -0,0 +1,4 @@ +export * from './scopes' +export * from './solver' +export * from './compiler' +export { printNodes } from './utils' \ No newline at end of file diff --git a/src/static-solver/scopes.ts b/src/static-solver/scopes.ts new file mode 100644 index 0000000..f31ce7f --- /dev/null +++ b/src/static-solver/scopes.ts @@ -0,0 +1,1399 @@ +import ts from 'typescript' +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import { failOnNode, getNodeLocation, isNonNullable } from './utils' +import { getLogger } from '../logging' +import { Mutable, isDeclared } from '../utils' + + +export interface Symbol { + readonly id: number + readonly name: string + readonly references: ts.Expression[] + readonly declaration?: ts.Node + readonly members: Map + readonly parent?: Symbol + readonly parentScope?: Scope + readonly computed?: boolean + readonly transient?: boolean // Used for "fake" symbols + readonly argSymbol?: Symbol + readonly isDeclared?: boolean // Checks for the `declare` keyword + + // These fields are used to simplify lookups of associated nodes + readonly importClause?: ts.ImportClause + readonly variableDeclaration?: ts.VariableDeclaration +} + +export interface Scope { + readonly id: number + readonly node: ts.Node + readonly symbol?: Symbol // Not all scopes have an associated symbol + readonly thisSymbol?: Symbol + readonly staticThisSymbol?: Symbol + readonly dependencies: Set + readonly parent?: Scope + readonly declarations: Map + readonly subscopes: Set + + // TODO: implement this to conditionally exclude code during synthesis + // readonly condition?: ts.Expression + + // Any call-like expression on external symbols are side-effects + // Any assignment/mutation to external symbols are side-effects + // readonly sideEffects: ts.Node[] +} + +export interface RootScope extends Scope { + readonly symbols: Map +} + +function isSymbol(obj: unknown): obj is Symbol { + return (!!obj && typeof obj === 'object' && 'id' in obj && 'name' in obj) +} + +function isStaticDeclaration(node: ts.Node) { + return ts.canHaveModifiers(node) && ts.getModifiers(node)?.find(x => x.kind === ts.SyntaxKind.StaticKeyword) +} + +export function getRootSymbol(sym: Symbol): Symbol { + while (sym.parent !== undefined) sym = sym.parent + + return sym +} + +export function getRootAndSuccessorSymbol(sym: Symbol): [Symbol, Symbol?] { + let successor: Symbol | undefined + while (sym.parent !== undefined) { + successor = sym + sym = sym.parent + } + + return [sym, successor] +} + +// Object literals have a class-like scope with a fixed receiver + +export function isScopeNode(node: ts.Node) { + switch (node.kind) { + case ts.SyntaxKind.Block: + case ts.SyntaxKind.CaseBlock: + // case ts.SyntaxKind.IfStatement: + case ts.SyntaxKind.ForStatement: + case ts.SyntaxKind.ForOfStatement: + case ts.SyntaxKind.SourceFile: + case ts.SyntaxKind.ArrowFunction: + case ts.SyntaxKind.Constructor: + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.GetAccessor: + case ts.SyntaxKind.SetAccessor: + case ts.SyntaxKind.EnumDeclaration: + case ts.SyntaxKind.ClassExpression: + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.FunctionExpression: + case ts.SyntaxKind.FunctionDeclaration: + return true + } +} + +// These are the only scopes that _might_ have symbol associated with them +function isNameableScopeNode(node: ts.Node) { + switch (node.kind) { + case ts.SyntaxKind.GetAccessor: + case ts.SyntaxKind.SetAccessor: + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.EnumDeclaration: + case ts.SyntaxKind.ClassExpression: + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.FunctionExpression: + case ts.SyntaxKind.FunctionDeclaration: + return true + } +} + +type FunctionLike = + | ts.ArrowFunction + | ts.AccessorDeclaration + | ts.FunctionExpression + | ts.FunctionDeclaration + | ts.MethodDeclaration + | ts.ConstructorDeclaration + +function isFunctionLike(node: ts.Node): node is FunctionLike { + switch (node.kind) { + case ts.SyntaxKind.GetAccessor: + case ts.SyntaxKind.SetAccessor: + case ts.SyntaxKind.Constructor: + case ts.SyntaxKind.ArrowFunction: + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.FunctionExpression: + case ts.SyntaxKind.FunctionDeclaration: + return true + } + + return false +} + +export function isDeclaration(node: ts.Node) { + switch (node.kind) { + // Ignore overloads + case ts.SyntaxKind.FunctionDeclaration: + return (node as any).body !== undefined + + case ts.SyntaxKind.Parameter: + case ts.SyntaxKind.Constructor: + case ts.SyntaxKind.PropertyDeclaration: + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.VariableDeclaration: + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.ImportDeclaration: + case ts.SyntaxKind.EnumDeclaration: + return true + } +} + +function isTransientSymbolNode(node: ts.Node) { + switch (node.kind) { + case ts.SyntaxKind.Block: + case ts.SyntaxKind.HeritageClause: + case ts.SyntaxKind.ForStatement: + case ts.SyntaxKind.ForOfStatement: + case ts.SyntaxKind.SourceFile: + case ts.SyntaxKind.ArrowFunction: + case ts.SyntaxKind.Constructor: + case ts.SyntaxKind.ImportDeclaration: + return true + + case ts.SyntaxKind.PropertyDeclaration: + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.VariableDeclaration: + case ts.SyntaxKind.ClassExpression: + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.FunctionExpression: + case ts.SyntaxKind.FunctionDeclaration: + if (!(node as any).name || !ts.isIdentifier((node as any).name)) { + return true + } + } +} + +let scopeCount = 0 + +function createDependencyGraph() { + // We track symbols per-file to ensure that ids stay consistent when the file doesn't change + let symbolCount = 0 + + const stack: Scope[] = [] + + function createSymbol(name: string, declaration: ts.Node | undefined, computed?: boolean): Symbol { + return { + id: symbolCount++, + name, + computed, + references: [], + declaration, + members: new Map(), + } + } + + function getScopeName(node: ts.Node): string { + if (ts.isIdentifier(node)) { + return node.text + } + + if (ts.isStringLiteral(node)) { + return node.text + } + + if (ts.isPropertyName(node)) { + if (ts.isComputedPropertyName(node)) { + return '__computed' + } + + return node.text // bug? handle numeric literals differently? + } + + if (ts.isVariableDeclaration(node) || ts.isParameter(node)) { + if (!ts.isIdentifier(node.name)) { + return `__bindingPattern_${symbolCount}` // XXX + //failOnNode('Binding pattern not implemented', node.name) + } + + return getScopeName(node.name) + } + + if (ts.isFunctionDeclaration(node) || ts.isFunctionExpression(node) || ts.isArrowFunction(node)) { + if (node.name) { + return getScopeName(node.name) + } + + return `__anonymousFunction_${symbolCount}` + } + + if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { + if (node.name) { + return getScopeName(node.name) + } + + return `__anonymousClass_${symbolCount}` + } + + if (ts.isEnumDeclaration(node)) { + return getScopeName(node.name) + } + + if (ts.isConstructorDeclaration(node)) { + return '__constructor' + } + + if (ts.isMethodDeclaration(node) || ts.isPropertyDeclaration(node)) { + return getScopeName(node.name) + } + + if (ts.isForStatement(node)) { + return `__forStatement_${symbolCount}` + } + + if (ts.isForOfStatement(node)) { + return `__forOfStatement_${symbolCount}` + } + + if (ts.isBlock(node) || ts.isCaseBlock(node)) { + return `__block_${symbolCount}` + } + + if (ts.isSourceFile(node)) { + return node.fileName + } + + if (ts.isNamespaceImport(node)) { + return node.name.text + } + + if (ts.isObjectLiteralExpression(node)) { + return `__object_${symbolCount}` + } + + if (ts.isHeritageClause(node)) { + return `super` + } + + if (ts.isGetAccessor(node)) { + return `__get_${getScopeName(node.name)}` + } + + if (ts.isSetAccessor(node)) { + return `__set_${getScopeName(node.name)}` + } + + if (ts.isBindingElement(node)) { + return getScopeName(node.name) + } + + failOnNode('Not supported', node) + } + + const symbols = new Map() + + function addReference(node: ts.Expression, symbol: Symbol) { + if (symbol.declaration === node || (symbol.declaration as any)?.name === node) { + symbols.set(node, symbol) + return + } + + symbol.references.push(node) + symbols.set(node, symbol) + } + + function isPrimaryExpressionLike(node: ts.Node) { + switch (node.kind) { + case ts.SyntaxKind.PropertyAccessExpression: + case ts.SyntaxKind.ElementAccessExpression: + case ts.SyntaxKind.ParenthesizedExpression: + return true + } + } + + function getStatements(node: ts.Node): readonly ts.Statement[] | undefined { + switch (node.kind) { + case ts.SyntaxKind.Block: + return (node as ts.Block).statements + // case ts.SyntaxKind.IfStatement: + // return (node as ts.IfStatement).elseStatement + // ? [(node as ts.IfStatement).thenStatement, (node as ts.IfStatement).elseStatement!] + // : [(node as ts.IfStatement).thenStatement] + case ts.SyntaxKind.ForStatement: + case ts.SyntaxKind.ForOfStatement: + return getStatements((node as ts.ForStatement | ts.ForOfStatement).statement) // BUG: I think this misses expression statements + case ts.SyntaxKind.SourceFile: + return (node as ts.SourceFile).statements + case ts.SyntaxKind.Constructor: + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.FunctionExpression: + case ts.SyntaxKind.FunctionDeclaration: + return (node as ts.FunctionDeclaration).body?.statements + case ts.SyntaxKind.CaseBlock: + return (node as ts.CaseBlock).clauses.flatMap(c => c.statements) + case ts.SyntaxKind.ArrowFunction: + return ts.isBlock((node as ts.ArrowFunction).body) + ? ((node as any).body as ts.Block).statements + : undefined + } + } + + function getDeclarations(node: ts.Node): ts.Node[] | undefined { + const declarations: ts.Node[] = [] + const statements = getStatements(node) + if (!statements) { + if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { + for (const e of node.members) { + if (isDeclaration(e)) { + declarations.push(e) + } + } + + return declarations + } else if (ts.isArrowFunction(node)) { + return [...node.parameters] + } + + return + } + + if (ts.isFunctionLike(node)) { + declarations.push(...node.parameters) + } + + if (ts.isForStatement(node) || ts.isForOfStatement(node)) { + if (node.initializer && ts.isVariableDeclarationList(node.initializer)) { + declarations.push(...node.initializer.declarations) + } + } + + for (const s of statements) { + if (isDeclaration(s)) { + declarations.push(s) + } else if (ts.isVariableStatement(s)) { + declarations.push(...s.declarationList.declarations) + } + } + + return declarations + } + + function addMember(target: Symbol, key: string | Symbol, value: Symbol) { + target.members.set(key, value) + ;(value as Mutable).parent = target + } + + function createThisSymbol(scope: Scope, val: ts.Node, isStatic?: boolean) { + const sym = createSymbol('this', val) + ;(sym as Mutable).parentScope = scope + + const target = isStatic ? scope.symbol! : getPrototypeSymbol(scope.symbol!) + for (const [k, v] of target.members) { + const member = createSymbol(v.name, v.declaration) + addMember(sym, k, member) + } + + return sym + } + + function getThisSymbol() { + let isStatic = false + for (let i = stack.length - 1; i >= 0; i--) { + const scope = stack[i] + const val = scope.symbol?.declaration + if (val && isStaticDeclaration(val)) { + isStatic = true + continue + } + + if (val === undefined || (!ts.isClassDeclaration(val) && !ts.isFunctionDeclaration(val) && !ts.isClassExpression(val) && !ts.isFunctionExpression(val))) { + continue + } + + if (isStatic) { + if (scope.staticThisSymbol !== undefined) { + return scope.staticThisSymbol + } + + const sym = createThisSymbol(scope, val, true) + ;(scope as Mutable).staticThisSymbol = sym + + return sym + } + + if (scope.thisSymbol !== undefined) { + return scope.thisSymbol + } + + const sym = createThisSymbol(scope, val, false) + ;(scope as Mutable).thisSymbol = sym + + return sym + } + + return createGlobalSymbol('this') + } + + function findSymbol(name: string): Symbol | undefined { + for (let i = stack.length - 1; i >= 0; i--) { + const scope = stack[i].symbol?.declaration + if (scope && (ts.isClassDeclaration(scope) || ts.isClassExpression(scope))) continue + + const symbol = stack[i].declarations.get(name) + if (symbol !== undefined) { + return symbol + } + } + } + + function getPrototypeSymbol(sym: Symbol) { + if (sym.members.has('prototype')) { + return sym.members.get('prototype')! + } + + const proto = createSymbol('prototype', undefined) + sym.members.set('prototype', proto) + ;(proto as Mutable).parent = sym + + return proto + } + + const scopeCache = new Map() + + function getScope(node: ts.Node): Scope { + if (scopeCache.has(node)) { + return scopeCache.get(node)! + } + + const dependencies = new Set() + const declarations = new Map() + const parentScope = stack[stack.length - 1] + + const scope: Scope = { + id: scopeCount++, + node, + declarations, + dependencies, + subscopes: new Set(), + parent: parentScope, + } + + parentScope.subscopes.add(scope) + scopeCache.set(node, scope) + + if (!isNameableScopeNode(node)) { + // if (node.kind === ts.SyntaxKind.IfStatement) { + // ;(scope as Mutable).condition = (node as ts.IfStatement).expression + // } + + return scope + } + + const symbol = bindSymbol(node, scope, ts.isClassElement(node)) + ;(scope as Mutable).symbol = symbol + + if (isDeclared(node)) { + ;(symbol as Mutable).isDeclared = true + } + + if (parentScope) { + const parentSym = parentScope.symbol + const parentVal = parentSym?.declaration + + if (parentVal && (ts.isClassDeclaration(parentVal) || ts.isClassExpression(parentVal))) { + const targetSym = isStaticDeclaration(node) ? parentSym : getPrototypeSymbol(parentSym) + addMember(targetSym, symbol.name, symbol) + } + } + + return scope + } + + function bindVariableDeclaration(decl: ts.VariableDeclaration) { + const symbols = ts.isObjectBindingPattern(decl.name) || ts.isArrayBindingPattern(decl.name) + ? visitBindingPattern(decl.name) + : [bindSymbol(decl)] + + for (const sym of symbols) { + (sym as Mutable).variableDeclaration = decl + } + } + + function visitParameterDeclaration(decl: ts.ParameterDeclaration, parent: ts.Node) { + if (ts.isParameterPropertyDeclaration(decl, parent)) { + const classScope = stack[stack.length - 2] + const symbol = bindSymbol(decl, classScope, true) + addMember(getPrototypeSymbol(classScope.symbol!), symbol.name, symbol) + } + + if (ts.isIdentifier(decl.name)) { + bindSymbol(decl) + } else { + visitBindingPattern(decl.name, true) + } + } + + function bindPropertyDeclaration(decl: ts.PropertyDeclaration, scope: Scope) { + const parentSym = scope.symbol! + const targetSym = isStaticDeclaration(decl) ? parentSym : getPrototypeSymbol(parentSym) + + if (!ts.isComputedPropertyName(decl.name)) { + const symbol = bindSymbol(decl, scope, true) + addMember(targetSym, symbol.name, symbol) + + return + } + + const symbol = visitExpression(decl.name.expression) + if (!symbol) { + failOnNode(`No symbol found for computed property name`, decl.name) + } + + // XXX + // parentSym.members.set(symbol, symbol) + } + + function visitScopeNode(node: ts.Node) { + const scope = getScope(node) + stack.push(scope) + + // init declarations first + const declarations = getDeclarations(node) + if (declarations) { + for (const decl of declarations) { + if (ts.isImportDeclaration(decl)) { + visitImportDeclaration(decl) + } else if (ts.isVariableDeclaration(decl)) { + bindVariableDeclaration(decl) + } else if (ts.isParameter(decl)) { + visitParameterDeclaration(decl, node) + } else if (ts.isPropertyDeclaration(decl)) { + bindPropertyDeclaration(decl, scope) + } else { + if (ts.isMethodDeclaration(decl)) { + if (ts.isComputedPropertyName(decl.name)) { + visitExpression(decl.name.expression) + } + } + const child = getScope(decl) // XXX + } + } + } + + if (isFunctionLike(node)) { + for (const param of node.parameters) { + if (param.initializer) { + visit(param.initializer) + } + } + + // Example case for why this is needed: + // `(a, b) => (id, ref) => b(id, ref, a)` + if (ts.isArrowFunction(node) && !ts.isBlock(node.body)) { + visit(node.body) + } else { + node.body?.forEachChild(visit) + } + } else { + // XXX: we add a fake scope for the heritage clause so it can be extracted more easily + const superClass = ts.isClassLike(node) + ? node.heritageClauses?.find(x => x.token === ts.SyntaxKind.ExtendsKeyword) + : undefined + + if (superClass) { + const s = getScope(superClass) + stack.push(s) + superClass.types.forEach(visit) + stack.pop() + if ((node as ts.ClassDeclaration | ts.ClassExpression).name) { + visit((node as ts.ClassDeclaration | ts.ClassExpression).name!) + } + ;(node as ts.ClassDeclaration | ts.ClassExpression).members.forEach(visit) + stack.pop() + + return + } + + node.forEachChild(visit) + } + + stack.pop()! + } + + function isTypeNode(node: ts.Node) { + switch (node.kind) { + case ts.SyntaxKind.HeritageClause: + return (node as ts.HeritageClause).token === ts.SyntaxKind.ImplementsKeyword + + case ts.SyntaxKind.Parameter: + return (node as ts.ParameterDeclaration).name.kind === ts.SyntaxKind.Identifier && + ((node as ts.ParameterDeclaration).name as ts.Identifier).text === 'this' + + case ts.SyntaxKind.VariableStatement: + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.ModuleDeclaration: + return !!ts.getModifiers(node as ts.ClassDeclaration | ts.VariableStatement | ts.ModuleDeclaration) + ?.find(m => m.kind === ts.SyntaxKind.DeclareKeyword) + + case ts.SyntaxKind.Constructor: + case ts.SyntaxKind.MethodDeclaration: + case ts.SyntaxKind.FunctionDeclaration: + return (node as ts.FunctionDeclaration | ts.MethodDeclaration | ts.ConstructorDeclaration).body === undefined + + case ts.SyntaxKind.ImportDeclaration: + return !!(node as ts.ImportDeclaration).importClause?.isTypeOnly + + case ts.SyntaxKind.ImportEqualsDeclaration: + return (node as ts.ImportEqualsDeclaration).isTypeOnly + + case ts.SyntaxKind.ExportDeclaration: + return (node as ts.ExportDeclaration).isTypeOnly + + case ts.SyntaxKind.ImportSpecifier: + case ts.SyntaxKind.ExportSpecifier: + return (node as ts.ImportSpecifier | ts.ExportSpecifier).isTypeOnly + + case ts.SyntaxKind.PropertySignature: + case ts.SyntaxKind.ConstructorType: + case ts.SyntaxKind.MappedType: + case ts.SyntaxKind.ConditionalType: + case ts.SyntaxKind.TypeLiteral: + case ts.SyntaxKind.FunctionType: + case ts.SyntaxKind.TypeAliasDeclaration: + case ts.SyntaxKind.InterfaceDeclaration: + case ts.SyntaxKind.TypeQuery: + case ts.SyntaxKind.TypeOperator: + case ts.SyntaxKind.TypeReference: + case ts.SyntaxKind.TypePredicate: + case ts.SyntaxKind.TypeParameter: + return true + } + + return false + } + + function visitIdentifier(node: ts.Identifier) { + if (ts.isJsxAttribute(node.parent) && node.parent.name === node) { + return + } + + // BIG HACK + // We're exploiting the fact that lowercase tags are intrinsic instead of fixing the real problem + if ((ts.isJsxOpeningElement(node.parent) || ts.isJsxClosingElement(node.parent) || ts.isJsxSelfClosingElement(node.parent)) && node.parent.tagName === node && node.text.toLowerCase() === node.text) { + return + } + + const name = node.text + + for (let i = stack.length - 1; i >= 0; i--) { + const sym = findSymbol(name) + + if (sym) { + addReference(node, sym) + + return sym + } + } + + const globalSym = createGlobalSymbol(name) + addReference(node, globalSym) + + return globalSym + } + + function visitThisExpression(node: ts.ThisExpression) { + const thisSymbol = getThisSymbol() + + addReference(node, thisSymbol) + + return thisSymbol + } + + function getMemberSymbol(target: Symbol, member: string | Symbol): Symbol { + if (target.members.has(member)) { + return target.members.get(member)! + } + + const name = typeof member === 'string' ? member : printSymbol(member) + const memberSym = createSymbol(name, undefined, typeof member !== 'string') + target.members.set(member, memberSym) + + return Object.assign(memberSym, { + parent: target, + argSymbol: typeof member !== 'string' ? member : undefined + }) + } + + function visitPropertyAccessExpression(node: ts.PropertyAccessExpression): Symbol | undefined { + const sym = visitExpression(node.expression) + if (!sym) { + return + } + + const name = node.name.text + const memberSymbol = getMemberSymbol(sym, name) + addReference(node, memberSymbol) + + return memberSymbol + } + + function visitElementAccessExpression(node: ts.ElementAccessExpression): Symbol | undefined { + const sym = visitExpression(node.expression) + const nameSym = visitExpression(node.argumentExpression) + if (!sym || !nameSym) { + return sym ?? nameSym + } + + const memberSymbol = getMemberSymbol(sym, nameSym) + addReference(node, memberSymbol) + + return memberSymbol + } + + function visitCallExpression(node: ts.CallExpression) { + const target = visitExpression(node.expression) + const args = node.arguments.map(visitExpression) + + // if (target && isSymbol(target)) { + // const graph = isSymbol(target) + // ? getGraphFromSymbol(target) + // : target + + // if (graph) { + // graph.sideEffects.push(node) + // } + + // symbols.set(node, target) + // } + + return undefined + } + + function visitNewExpression(node: ts.NewExpression) { + const target = visitExpression(node.expression) + const args = node.arguments?.map(visitExpression) ?? [] + + // if (target && !isSymbolWithinCurrentGraph(target)) { + // const graph = isSymbol(target) + // ? getGraphFromSymbol(target) + // : target + + // if (graph) { + // graph.sideEffects.push(node) + // } + // } + + return undefined + } + + function visitEnumMember(node: ts.EnumMember) { + if (!ts.isIdentifier(node.name)) { + failOnNode('Not implemented', node) + } + + const currentScope = stack[stack.length - 1] + const memberSymbol = createSymbol(node.name.text, node) + currentScope.symbol!.members.set(node.name.text, Object.assign(memberSymbol, { + parent: currentScope.symbol + })) + + return memberSymbol + } + + function visitBinaryExpression(node: ts.BinaryExpression) { + const left = visitExpression(node.left) + const right = visitExpression(node.right) + + switch (node.operatorToken.kind) { + case ts.SyntaxKind.EqualsToken: + case ts.SyntaxKind.PlusEqualsToken: + case ts.SyntaxKind.MinusEqualsToken: + case ts.SyntaxKind.AsteriskAsteriskEqualsToken: + case ts.SyntaxKind.AsteriskEqualsToken: + case ts.SyntaxKind.SlashEqualsToken: + case ts.SyntaxKind.PercentEqualsToken: + case ts.SyntaxKind.AmpersandEqualsToken: + case ts.SyntaxKind.BarEqualsToken: + case ts.SyntaxKind.CaretEqualsToken: + case ts.SyntaxKind.LessThanLessThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.BarBarEqualsToken: + case ts.SyntaxKind.AmpersandAmpersandEqualsToken: + case ts.SyntaxKind.QuestionQuestionEqualsToken: { + // const target = isSymbol(left) ? getGraphFromSymbol(left) : left + // if (target && target !== stack[stack.length - 1]) { + // target.sideEffects.push(node) + // } + } + } + + return undefined + } + + function maybeAddDependency(node: ts.Node, sym: Symbol) { + const parent = node.parent + if (isPrimaryExpressionLike(parent)) { + return + } + + const currentScope = stack[stack.length - 1] + // This is a terminal node, add it to the dependency graph + if (currentScope.symbol !== sym) { + currentScope.dependencies.add(sym) + } + + // Check for any computed symbols in intermediate expressions + let currentSym: Symbol | undefined = sym + while (currentSym !== undefined) { + const argSymbol = currentSym.computed ? currentSym.argSymbol : undefined + if (argSymbol) { + if (currentScope.symbol !== argSymbol) { + currentScope.dependencies.add(argSymbol) + } + } + currentSym = currentSym.parent + } + } + + // Expressions should result in a graph or symbol + function visitExpression(node: ts.Expression): Symbol | undefined { + if (isTypeNode(node)) { + return + } + + if (isScopeNode(node)) { + return void visitScopeNode(node) + } + + function fn(): Symbol | undefined { + switch (node.kind) { + case ts.SyntaxKind.Identifier: + return visitIdentifier(node as ts.Identifier) + case ts.SyntaxKind.ThisKeyword: + return visitThisExpression(node as ts.ThisExpression) + case ts.SyntaxKind.PropertyAccessExpression: + return visitPropertyAccessExpression(node as ts.PropertyAccessExpression) + case ts.SyntaxKind.ElementAccessExpression: + return visitElementAccessExpression(node as ts.ElementAccessExpression) + case ts.SyntaxKind.CallExpression: + return visitCallExpression(node as ts.CallExpression) + case ts.SyntaxKind.NewExpression: + return visitNewExpression(node as ts.NewExpression) + case ts.SyntaxKind.BinaryExpression: + return visitBinaryExpression(node as ts.BinaryExpression) + case ts.SyntaxKind.AwaitExpression: + case ts.SyntaxKind.ParenthesizedExpression: + return visitExpression((node as ts.ParenthesizedExpression).expression) + default: + node.forEachChild(visit) + } + } + + const symbol = fn() + if (symbol !== undefined) { + maybeAddDependency(node, symbol) + + return symbol + } + } + + function createGlobalSymbol(name: string) { + const currentScope = stack[stack.length - 1] + const symbol = createSymbol(name, undefined) + stack[0].declarations.set(name, symbol) + currentScope.dependencies.add(symbol) + ;(symbol as Mutable).parentScope = stack[0] + + return symbol + } + + function visitImportDeclaration(node: ts.ImportDeclaration) { + const clause = node.importClause + if (!clause) { + // side-effect inducing + return + } + + function bindWithClause(node: ts.Node) { + const sym = bindSymbol(node) + ;(sym as Mutable).importClause = clause + } + + const bindings = clause.namedBindings + if (bindings) { + if (ts.isNamespaceImport(bindings)) { + bindWithClause(bindings.name) + } else { + bindings.elements.forEach(e => { + bindWithClause(e.name) + }) + } + } + + if (clause.name) { + bindWithClause(clause.name) + } + } + + function bindSymbol(node: ts.Node, parentScope = stack[stack.length - 1], isClassElement = false) { + const name = getScopeName(node) + const symbol = createSymbol(name, node) + symbols.set(node, symbol) + ;(symbol as Mutable).parentScope = parentScope + if (!isClassElement) { + stack[stack.length - 1].declarations.set(name, symbol) + } + + return symbol + } + + function visitBindingPattern(node: ts.BindingPattern, visitInitializer = false) { + const symbols: Symbol[] = [] + for (const element of node.elements) { + if (!ts.isBindingElement(element)) { + continue + } + + if (ts.isIdentifier(element.name)) { + symbols.push(bindSymbol(element)) + + if (visitInitializer && element.initializer) { + visitExpression(element.initializer) + } + } else { + visitBindingPattern(element.name, visitInitializer) + } + } + + return symbols + } + + function visitCatchClause(node: ts.CatchClause) { + if (!node.variableDeclaration) { + return visitScopeNode(node.block) + } + + const scope = getScope(node.block) + stack.push(scope) + bindSymbol(node.variableDeclaration) + node.block.forEachChild(visit) + stack.pop() + } + + function visitExportDeclaration(node: ts.ExportDeclaration) { + if (!node.exportClause || !ts.isNamedExports(node.exportClause)) { + return + } + + for (const spec of node.exportClause.elements) { + // We only want to add a symbol for the local identifier + const localIdent = spec.propertyName ?? spec.name + visitExpression(localIdent) + } + } + + function visit(node: ts.Node) { + if (isTypeNode(node)) { + return + } + + if (isScopeNode(node)) { + // XXX: this leaks the symbols into the outer scope. We're assuming that the method decl is apart of an object literal exp. + if (node.kind === ts.SyntaxKind.MethodDeclaration && (node as any).name.kind === ts.SyntaxKind.ComputedPropertyName && node.parent.kind === ts.SyntaxKind.ObjectLiteralExpression) { + visitExpression((node as any).name.expression) + } + + return void visitScopeNode(node) + } + + if (ts.isPropertyAssignment(node)) { + if (ts.isComputedPropertyName(node.name)) { + visitExpression(node.name.expression) + } + + return void visit(node.initializer) + } + + if (ts.isPropertyDeclaration(node)) { + return node.initializer ? void visit(node.initializer) : void 0 + } + + if (ts.isExpression(node)) { + return void visitExpression(node) + } + + if (ts.isExpressionStatement(node)) { + return void visitExpression(node.expression) + } + + if (ts.isEnumMember(node)) { + return void visitEnumMember(node) + } + + if (ts.isCatchClause(node)) { + return void visitCatchClause(node) + } + + if (ts.isExportDeclaration(node)) { + return void visitExportDeclaration(node) + } + + if (ts.isLabeledStatement(node)) { + return void visit(node.statement) + } + + // Skip labels + if (ts.isBreakOrContinueStatement(node)) { + return + } + + // This will be visited earlier + if (ts.isImportDeclaration(node)) { + return + } + + node.forEachChild(visit) + } + + return (s: ts.Node) => { + stack.push({ + symbols, + symbol: createSymbol('__global', undefined), + declarations: new Map(), + dependencies: new Set(), + subscopes: new Set(), + } as RootScope) + + visitScopeNode(s) + + return stack.pop()! as RootScope + } +} + +// What do I want to know? +// Given a function/module/class, determine: +// 1. The permissions required to execute the code +// 2. The resources required to execute the code +// 3. The symbolic dependencies + +// var a, b; +// var e = {foo: 5, bar: 6, baz: ['Baz', 'Content']}; +// var arr = []; +// ({baz: [arr[0], arr[3]], foo: a, bar: b} = e); +// getTerminalLogger().log(a + ',' + b + ',' + arr); // displays: 5,6,Baz,,,Content +// [a, b] = [b, a]; // swap contents of a and b + + +// isSourceFileDefaultLibrary(file: SourceFile): boolean; + +const refInScopeCache = new Map() +export function getReferencesInScope(symbol: Symbol, scope: Scope) { + const key = `${scope.id}:${symbol.id}` + if (refInScopeCache.has(key)) { + return refInScopeCache.get(key)! + } + + const result: ts.Node[] = [] + + function isInScope(node: ts.Node) { + if (ts.findAncestor(node, n => n === scope.node)) { + return true + } + + return false + } + + for (const ref of symbol.references) { + if (isInScope(ref)) { + result.push(ref) + } + } + + refInScopeCache.set(key, result) + + return result +} + +/** Checks if `b` is contained by `a` */ +function isSubscope(a: Scope, b: Scope) { + let c: Scope | undefined = b + + do { + if (a === c) return true + c = c.parent + } while (c !== undefined) + + return false +} + +function getChildrenDeps(scope: Scope, excluded: Scope[] = []): Symbol[] { + const deps: Symbol[] = [] + for (const v of scope.subscopes) { + if (excluded.includes(v)) continue + + deps.push( + ...v.dependencies, + ...getChildrenDeps(v, excluded) + ) + } + + return deps +} + +export function getContainingScope(symbol: Symbol): Scope { + const scope = getRootSymbol(symbol).parentScope + if (!scope) { + if (symbol.declaration) { + failOnNode('Symbol is not apart of a graph', symbol.declaration) + } + + throw new Error(`Symbol is not apart of a graph: ${symbol.name}`) + } + + return scope +} + + +export function getImmediatelyCapturedSymbols(scope: Scope, excluded: Scope[] = []) { + if (!scope.node) { // Global scope + return [] + } + + const symbols: Symbol[] = [] + const deps = [...getChildrenDeps(scope, excluded), ...scope.dependencies] + for (const d of deps) { + if (!isSubscope(scope, getContainingScope(d))) { + symbols.push(d) + } + } + + return symbols +} + +export function isAssignmentExpression(node: ts.BinaryExpression) { + switch (node.operatorToken.kind) { + case ts.SyntaxKind.EqualsToken: + case ts.SyntaxKind.PlusEqualsToken: + case ts.SyntaxKind.MinusEqualsToken: + case ts.SyntaxKind.AsteriskAsteriskEqualsToken: + case ts.SyntaxKind.AsteriskEqualsToken: + case ts.SyntaxKind.SlashEqualsToken: + case ts.SyntaxKind.PercentEqualsToken: + case ts.SyntaxKind.AmpersandEqualsToken: + case ts.SyntaxKind.BarEqualsToken: + case ts.SyntaxKind.CaretEqualsToken: + case ts.SyntaxKind.LessThanLessThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.BarBarEqualsToken: + case ts.SyntaxKind.AmpersandAmpersandEqualsToken: + case ts.SyntaxKind.QuestionQuestionEqualsToken: + return true + } + + return false +} + + +// Rules: +// 1. Any symbol on the LHS of an assignment operation must be decomposed and passed by reference +// * Mutation of variables shared between multiple functions cannot be captured without transforming it into a binding +// 2. Computed symbols can only be captured in their entirety if they are constant +// 3. References to private members result in an indivisible function. Additional bindings need to be added to make the function divisible. + +// TODO: add annotations to methods so they can be captured + +// Transforms a graph into a function declaration that passes in captured symbols by argument +// Import declarations are left outside of the expression, function/class declarations are placed +// inside, and stateful declarations are made into arguments + +export function liftScope(scope: Scope, capturedGlobals?: string[], excluded: Scope[] = []) { + const capturedSymbols = new Set() + const globals = new Set() + + const captured = getImmediatelyCapturedSymbols(scope, excluded) + for (const c of captured) { + const rootSym = getRootSymbol(c) + const val = rootSym.declaration + if (val === undefined) { + if (capturedGlobals && capturedGlobals.includes(rootSym.name)) { + globals.add(rootSym) + } else { + // outerScopes.add(rootScope) + } + continue + } + + capturedSymbols.add(c) + } + + return { + globals: Array.from(globals), + captured: Array.from(capturedSymbols), + } +} + +export function unwrapScope(scope: Scope): ts.Node | undefined { + const decl = scope.symbol?.declaration + if (decl && ts.isHeritageClause(decl)) { + return decl.types[0].expression + } + + return decl +} + +export function isParameter(scope: Scope): boolean { + const val = unwrapScope(scope) + if (val === undefined) { + return false + } + + return ts.isParameter(val) +} + +export function getSubscopeDfs(scope: Scope, node: ts.Node): Scope | undefined { + for (const g of scope.subscopes.values()) { + const n = getSubscopeDfs(g, node) + if (n) return n + } + + if (ts.findAncestor(node, n => n === scope.node)) { + return scope + } + + return +} + +export function getSubscopeContaining(scope: Scope, node: ts.Node) { + for (const g of scope.subscopes.values()) { + if (ts.findAncestor(node, n => n === g.node)) { + return g + } + } + + failOnNode('No subscope found', node) +} + +export function createGraph(node: ts.Node): RootScope { + return createDependencyGraph()(node) +} + +export function createGraphOmitGlobal(node: ts.Node): Scope { + return getSubscopeContaining(createDependencyGraph()(node), node) +} + +export function printSymbol(symbol: Symbol): string { + if (!symbol.parent) { + return symbol.name + } + + if (symbol.computed) { + return `${printSymbol(symbol.parent)}[${symbol.name}]` + } + + return `${printSymbol(symbol.parent)}.${symbol.name}` +} + +export function printDependencies(scope: Scope) { + return [...scope.dependencies].map(printSymbol).join(', ') +} + +function getScopeSymbol(scope: Scope) { + while (!scope.symbol && scope.parent) { + scope = scope.parent + } + + if (!scope.symbol) { + throw new Error(`No scope found with symbol starting from scope: ${scope}`) + } + + return scope.symbol +} + +function printGraph(scope: Scope, maxDepth = Infinity, hideAmbient = false, depth = 0) { + if (depth >= maxDepth) return + + const print = (s: string) => getLogger().log(`${' '.repeat(depth)}${s}`) + // const immediateDeps = [...graph.dependencies, ...getChildrenDeps(graph)] + const deps = printDependencies(scope) + const sym = getScopeSymbol(scope) + const val = sym.declaration + const isAmbient = val=== undefined + + if (isAmbient && hideAmbient && depth > 0) return + + if (!isAmbient && ts.isParameter(val)) { + print(' ' + sym.name + (deps ? ` [${deps}]` : '')) + } else { + print(sym.name + (isAmbient ? '*' : '') + (deps ? ` [${deps}]` : '')) + } + + for (const [k, v] of scope.subscopes.entries()) { + printGraph(v, maxDepth, hideAmbient, depth + 1) + } +} + +export function createGraphFromText(fileName: string, text: string) { + const sourceFile = ts.createSourceFile(fileName, text, { languageVersion: ts.ScriptTarget.ES2020 }, true) + + return createDependencyGraph()(sourceFile) +} + +export function createGraphFromFile(sourceFile: ts.SourceFile) { + return createDependencyGraph()(sourceFile) +} + +// Kahn's algorithm +// e[0] is from +// e[1] is to +export function topoSort(edges: [T, T][]) { + const l: T[] = [] + const s = Array.from(getRootNodes(edges)) + + while (s.length > 0) { + const n = s.pop()! + l.push(n) + + for (const m of extractEdges(n, edges)) { + if (!edges.find(e => e[1] === m)) { + s.push(m) + } + } + } + + if (edges.length !== 0) { + throw new Error('Cycle detected') + } + + return l +} + +function* extractEdges(n: T, edges: [T, T][]) { + for (let i = edges.length - 1; i >= 0; i--) { + if (n === edges[i][0]) { + yield edges.splice(i, 1)[0][1] + } + } +} + +function* getRootNodes(edges: [T, T][]) { + const nodes = new Set() + const inc = new Set() + + for (const e of edges) { + nodes.add(e[0]) + nodes.add(e[1]) + inc.add(e[1]) + } + + for (const n of nodes) { + if (!inc.has(n)) { + yield n + } + } +} + + diff --git a/src/static-solver/solver.ts b/src/static-solver/solver.ts new file mode 100644 index 0000000..86ff773 --- /dev/null +++ b/src/static-solver/solver.ts @@ -0,0 +1,1302 @@ +import ts from 'typescript' +import * as assert from 'node:assert' +import { failOnNode, getNodeLocation } from './utils' +import { getLogger } from '../logging' +import { isNode } from '../utils' + +function isExported(node: ts.Node) { + return ts.canHaveModifiers(node) && !!ts.getModifiers(node)?.find(m => m.kind === ts.SyntaxKind.ExportKeyword) +} + +function isUsing(node: ts.VariableStatement) { + return (node.declarationList.flags & ts.NodeFlags.Using) === ts.NodeFlags.Using || + (node.declarationList.flags & ts.NodeFlags.AwaitUsing) === ts.NodeFlags.AwaitUsing +} + +type SubstitutionFunction = (node: ts.Node, scopes: ts.Node[]) => any + +const internalBrand = Symbol.for('__internalFunction') +const unknown = Symbol.for('unknown') +const uninitialized = Symbol.for('uninitialized') +const evaluatorSymbol = Symbol.for('lazyEvaluator') +const typeSymbol = Symbol.for('__type') +const unionSymbol = Symbol.for('union') + +export function isInternalFunction(fn: (...args: any[]) => unknown): fn is typeof fn & { [internalBrand]: InternalFunctionData } { + return (typeof fn === 'function' || (typeof fn === 'object' && fn !== null)) && internalBrand in fn +} + +interface InternalFunctionData { + length: number + getSource: () => string +} + +export function createInternalFunction(fn: T, length?: number, getSource?: () => string): T { + return ((fn as any)[internalBrand] = { length, getSource }, fn) +} + +export function getFunctionLength(fn: any) { + if (isInternalFunction(fn)) { + return fn[internalBrand].length + } + + return fn.length +} + +export function getSourceCode(fn: any) { + if (isInternalFunction(fn)) { + return fn[internalBrand].getSource() + } + + return fn.toString() +} + +function isInternalThis(thisArg: any): thisArg is ts.Node[] { + return Array.isArray(thisArg) && (thisArg.length === 0 || isNode(thisArg[0])) +} + +export function isUnknown(val: any): val is typeof unknown { + return val === unknown +} + +export function createUnknown() { + return unknown +} + +export function wrapType(type: ts.TypeNode) { + return { [typeSymbol]: type } +} + +export function createUnion(values: Iterable | (() => Iterable)) { + if (typeof values === 'function') { + return { + [unionSymbol]: true, + [Symbol.iterator]: values, + } + + } + + return { [unionSymbol]: true, [Symbol.iterator]: values[Symbol.iterator].bind(values) } +} + +export function isUnion(val: unknown): val is ({ [unionSymbol]: true } & Iterable) { + return typeof val === 'object' && !!val && unionSymbol in val +} + +function lazy(fn: (...args: T) => U): ((...args: T) => U) & { [evaluatorSymbol]: any } { + return Object.assign(fn, { [evaluatorSymbol]: true }) +} + +function isLazy(val: any): val is ((...args: any[]) => any) & { [evaluatorSymbol]: any } { + return !!val && typeof val === 'function' && evaluatorSymbol in val +} + +export function evaluate(val: any): any { + if (!!val) { + if (typeof val === 'function' && evaluatorSymbol in val) { + return evaluate(val()) + } else if (isUnion(val)) { + return createUnion(Array.from(val).map(evaluate)) + } + } + + return val +} + +class _Array extends Array { + join(sep: any) { + if (this.some(x => isUnknown(x) || x === uninitialized)) { + return createUnknown() as any + } + + return super.join(sep) + } + + splice(start: number, deleteCount?: number, items?: any[]) { + if (isUnknown(start) || isUnknown(deleteCount)) { + return [createUnknown()] as any[] + } + + return !items ? super.splice(start, deleteCount) : super.splice(start, deleteCount!, items) + } +} + +export function createStaticSolver( + substitute?: SubstitutionFunction +) { + // XXX: only used to check for recursion + const callStack: [node: ts.Node, args: any[]][] = [] + + function createSolver( + scopes: ts.Node[] = [], + state = new Map(), + context?: SubstitutionFunction + ): { solve: (node: ts.Node) => any, getState: () => Map, printState: () => void } { + if (scopes.length === 0 && !state.has('exports')) { + state.set('exports', {}) + } + + function printState(s = state) { + for (const [k, v] of s.entries()) { + if (isLazy(v)) { + getLogger().log(`${k}: [lazy]`) + } else { + const val = typeof v === 'object' ? JSON.stringify(v) : typeof v === 'symbol' ? `[${v.description}]` : v + getLogger().log(`${k}: ${val}`) + } + } + } + + function getSubstitute(node: ts.Node, scopes: ts.Node[]): any { + return evaluate(fn()) + + function fn() { + if (ts.isIdentifier(node)) { + if (state.has(node.text)) { + return state.get(node.text) + } + } else if (node.kind === ts.SyntaxKind.ThisKeyword && state.has('this')) { + return state.get('this') + } + + if (context) { + const sub = context(node, scopes) + if (sub === node) { + failOnNode(`Found recursive call`, node) + } + + return isNode(sub) ? solve(sub) : sub + } + + return unknown + } + } + + function solvePropertyName(node: ts.PropertyName) { + if (ts.isComputedPropertyName(node)) { + const solved = solve(node.expression) + + // XXX + return typeof solved === 'string' ? solved : undefined + } + + return node.text + } + + function solveMemberName(node: ts.MemberName) { + return node.text + } + + function solveObjectLiteral(node: ts.ObjectLiteralExpression) { + const result: Record = {} + for (const v of node.properties) { + if (ts.isSpreadAssignment(v)) { + const val = solve(v.expression) + if (val === unknown) { + return val + } + + if (isUnion(val)) { + return unknown + } + + if (val !== uninitialized) { + if (typeof val !== 'object' && typeof val !== 'undefined') { + // TODO: need to do CFA in the containing declaration to determine + // if this is a compiler bug or a user bug. The value might not be an + // object if the spread expression is guarded by a `typeof` check + // failOnNode(`Expected an object, got type "${typeof val}"`, node) + return unknown + } else { + Object.assign(result, val) + } + } + } else { + if (ts.isMethodDeclaration(v) || ts.isAccessor(v)) { + throw new Error('method/accessor decl not supported') + } + if (ts.isShorthandPropertyAssignment(v)) { + result[v.name.text] = solve(v.name) + } else { + if (ts.isComputedPropertyName(v.name)) { + const val = solve(v.name.expression) + result[val] = solve(v.initializer) + } else { + result[v.name.text] = solve(v.initializer) + } + } + } + } + + return result + } + + function solveIdentifier(node: ts.Identifier): any { + if (node.text === 'undefined') { + return undefined + } + + return getSubstitute(node, scopes) + } + + function solveThisKeyword(node: ts.Node): any { + return getSubstitute(node, scopes) + } + + function access(target: any, arg: any, questionDotToken = false): any { + if (isUnknown(target)) { + return unknown + } + + if (isUnion(target)) { + return createUnion(function* () { + for (const t of target) { + yield access(t, arg, questionDotToken) + } + }) + } + + if (isUnion(arg)) { + return createUnion(function* () { + for (const key of arg) { + yield access(target, key, questionDotToken) + } + }) + } + + if (isUnknown(arg)) { + if (isLazy(target)) { + return lazy(() => createUnion(Object.values(target()))) + } else { + return target ? createUnion(Object.values(target)) : undefined + } + } + + // FIXME: how to make sure we don't evaluate blocks before var init (TDZ?) + return target?.[arg] + + if (questionDotToken) { + return target?.[arg] + } else { + return target[arg] + } + } + + function solvePropertyAccess(node: ts.PropertyAccessExpression): any { + assert.ok(!ts.isPrivateIdentifier(node.name)) + const target = solve(node.expression) + + return access(target, node.name.text, !!node.questionDotToken) + } + + function solveElementAccessExpression(node: ts.ElementAccessExpression): any { + const target = solve(node.expression) + const arg = solve(node.argumentExpression) + + return access(target, arg, !!node.questionDotToken) + } + + function getThisArg(exp: ts.CallExpression): any { + if (ts.isPropertyAccessExpression(exp.expression) || ts.isElementAccessExpression(exp.expression)) { + return solve(exp.expression.expression) + } + } + + // TODO: merge w/ InterfaceDeclaration + function solveClassDeclarationOrExpression(node: ts.ClassDeclaration | ts.ClassExpression): any { + // TODO: static members + const ctor = node.members.find(ts.isConstructorDeclaration) + const fields = node.members.filter(ts.isPropertyDeclaration) + const methods = node.members.filter(ts.isMethodDeclaration) + const heritage = node.heritageClauses?.filter(x => x.token === ts.SyntaxKind.ExtendsKeyword).map(x => x.types[0].expression)[0] + const proto: any = {} + + // `this` is the execution scope + const c = createInternalFunction(function (this: ts.Node[], thisArg: any, ...args: any[]) { + if (heritage) { + const fn = solve(heritage) + if (typeof fn === 'function') { + thisArg ??= {} + if (isInternalFunction(fn)) { + thisArg = fn.call(this, thisArg, ...args) ?? thisArg + } else { + thisArg = fn.call(thisArg, ...args) ?? thisArg + } + } else { + if (fn !== unknown) { + failOnNode(`Super class was not a function`, heritage) + } + } + } + + //thisArg.constructor = c + + // methods... + for (const [k, v] of Object.entries(proto)) { + thisArg[k] = v + } + + for (const f of fields) { + // These don't have the correct scope? + const name = solvePropertyName(f.name) + if (name !== undefined) { + thisArg[name] = f.initializer ? solve(f.initializer) : undefined + } + } + + if (ctor != undefined) { + const fn = solve(ctor) + if (typeof fn === 'function') { + if (isInternalFunction(fn)) { + thisArg = fn.call(this, thisArg, ...args) ?? thisArg + } else { + thisArg = fn.call(thisArg, ...args) ?? thisArg + } + } else { + if (fn !== unknown) { + failOnNode(`Constructor was not a function`, ctor) + } + } + } + + return thisArg + }, ctor?.parameters.length, () => node.getText(node.getSourceFile())) + + for (const m of methods) { + if (m.modifiers?.find(x => x.kind === ts.SyntaxKind.StaticKeyword)) { + // TODO: handle static + } else { + const name = solvePropertyName(m.name) + if (name !== undefined) { + if (typeof name !== 'string') { + failOnNode(`Name is not a string`, m) + } + proto[name] = solveFunctionLikeDeclaration(m) + } + } + } + + if (node.name) { + Object.defineProperty(c, 'name', { + value: node.name.text, + configurable: true, + }) + } + + c.prototype = proto + + return c + } + + function solveArguments(nodes: ts.Expression[] | ts.NodeArray = []) { + const results: any[] = [] + for (const n of nodes) { + if (ts.isSpreadElement(n)) { + const val = solve(n.expression) + if (val === unknown) { + results.push(unknown) + } else if (val) { + results.push(...val) + } + } else { + results.push(solve(n)) + } + } + + return results + } + + // XXX: only used for recursion + function callWithStack(...args: Parameters): any { + if (callStack.length > 50) { + // getLogger().warn('Potentially recursive function call, returning "unknown" type') + return createUnknown() + } + + const isRecursive = !!callStack.find(f => f[0] === args[0] && f[1].map((e, i) => e === args[4][i]).reduce((a, b) => a && b, true)) + if (isRecursive) { + return createUnknown() + } + + callStack.push([args[0], args[4]]) + // try { + const res = call(...args) + callStack.pop() + + return res + // } catch (e) { + // // XXX + // return unknown + // } + } + + function call(source: ts.Node, scopes: ts.Node[], fn: any, thisArg: any, args: any[]): any { + if (isUnion(fn)) { + return createUnion(function* () { + for (const f of fn) { + if (isUnknown(f)) { + yield f + // failOnNode('Found unknown symbol in union', source) + } else { + yield callWithStack(source, scopes, f, thisArg, args) + } + } + }) + } + + if (typeof fn !== 'function') { + failOnNode(`Not a function: ${fn}`, source) + } + + if (isInternalFunction(fn)) { + return fn.call(scopes, thisArg, ...args) + } else { + if (ts.isNewExpression(source)) { + return Reflect.construct(fn, args) + } else { + return fn.call(thisArg, ...args) + } + } + } + + function solveCallExpression(node: ts.CallExpression) { + const args = solveArguments(node.arguments) + const thisArg = getThisArg(node) + const fn = solve(node.expression) + if (fn === unknown || thisArg === unknown || !fn) { + return unknown + } + + return callWithStack(node, scopes, fn, thisArg, args) + } + + function solveNewExpression(node: ts.NewExpression) { + const args = solveArguments(node.arguments) + const fn = solve(node.expression) + if (fn === unknown) { + return unknown + } + + return callWithStack(node, scopes, fn, {}, args) + } + + function solveSuperExpression(node: ts.SuperExpression) { + return unknown + } + + function solveBindingName(node: ts.BindingName) { + if (ts.isIdentifier(node)) { + return node.text + } else { + failOnNode('Not implemented', node) + } + } + + const usingStateKey = '__kUsingState__' + function getUsingState() { + if (!state.has(usingStateKey)) { + state.set(usingStateKey, []) + } + + return state.get(usingStateKey)! as any[] + } + + function solveUsingState() { + if (!state.has(usingStateKey)) { + return + } + + for (const r of getUsingState().map(evaluate)) { + if (isUnknown(r)) continue + + if (Symbol.dispose in r) { + r[Symbol.dispose]() + } else if (Symbol.asyncDispose in r) { + r[Symbol.asyncDispose]() + } else { + throw new Error(`Missing dispose method in resource`) + } + } + } + + function setLazyState(name: string, isExported: boolean, fn: () => any) { + const result = lazy(() => { + const val = fn() + state.set(name, val) + if (isExported && state.has('exports')) { + state.get('exports')![name] = val + } + return val + }) + state.set(name, result) + if (isExported && state.has('exports')) { + state.get('exports')![name] = result + } + + return result + } + + function solveExpressionStatement(node: ts.ExpressionStatement) { + const exp = node.expression + if (ts.isBinaryExpression(exp) && (exp.operatorToken.kind === ts.SyntaxKind.EqualsToken || exp.operatorToken.kind === ts.SyntaxKind.QuestionQuestionEqualsToken)) { + const left = exp.left + const right = exp.right + + if (ts.isPropertyAccessExpression(left)) { + const target = solve(left.expression) + if (isUnknown(target) || isUnion(target)) { + return + } + if (target === undefined || (typeof target !== 'object' && typeof target !== 'function')) { + failOnNode(`Not an object: ${target}`, node) + } + const memberName = solveMemberName(left.name) + if (typeof memberName !== 'string') { + failOnNode(`Not a string: ${memberName}`, node) + } + + if (exp.operatorToken.kind === ts.SyntaxKind.QuestionQuestionEqualsToken) { + target[memberName] ??= solve(right) + } else { + target[memberName] = solve(right) + } + } else if (ts.isIdentifier(left)) { + if (!state.has(left.text)) { + const val = solve(left) + if (val === undefined) { + failOnNode(`Identifier "${left.text}" has not been declared`, node) + } + } + // FIXME: this only sets the scoped state? + if (exp.operatorToken.kind === ts.SyntaxKind.QuestionQuestionEqualsToken) { + state.set(left.text, evaluate(state.get(left.text)) ?? solve(right)) + } else { + state.set(left.text, solve(right)) + } + } else if (ts.isObjectBindingPattern(left)) { + const rightVal = solve(right) + for (const element of left.elements) { + const localName = solveBindingName(element.name) + const targetName = element.propertyName ? solvePropertyName(element.propertyName) : localName + if (!targetName) { + failOnNode('No target name', left) + } + + if (rightVal === unknown) { + state.set(localName, unknown) + } else { + const val = rightVal?.[targetName] ?? (element.initializer ? solve(element.initializer) : unknown) + state.set(localName, val) + } + } + } else if (ts.isElementAccessExpression(left)) { + // TODO + solve(left) + solve(right) + } + } else { + // XXX: used to dump logs + if (ts.isCallExpression(exp) && ts.isPropertyAccessExpression(exp.expression) && ts.isIdentifier(exp.expression.expression) && exp.expression.expression.text === 'console') { + evaluate(solve(exp)) + } + + // getTerminalLogger().log(getNodeLocation(node), node.pos) + setLazyState(`__expression:${node.pos}`, false, () => solve(exp)) + } + } + + function getProp(val: any, prop: string | number) { + if (val === unknown) { + return val + } + + if (val === undefined) { + return unknown + } + + return val[prop] + } + + function solveBindingPattern(node: ts.BindingPattern, lazyGet: () => any, isExported: boolean) { + for (let i = 0; i < node.elements.length; i++) { + const element = node.elements[i] + if (ts.isBindingElement(element)) { + if (ts.isIdentifier(element.name)) { + const name = element.name.text + setLazyState(name, isExported, () => getProp(lazyGet(), name)) + } else { + solveBindingPattern(element.name, () => getProp(lazyGet(), i), isExported) + } + } + } + } + + function solveStatement(node: ts.Statement) { + if (ts.isClassDeclaration(node)) { + if (!node.name) { + failOnNode('No name', node) + } + setLazyState(node.name.text, isExported(node), () => solve(node)) + } else if (ts.isFunctionDeclaration(node)) { + if (!node.name) { + failOnNode('No name', node) + } + setLazyState(node.name.text, isExported(node), () => solve(node)) + } else if (ts.isVariableStatement(node)) { + const solver = createSolver([...scopes, node], undefined, getSubstitute) + + // XXX: only handles 1 decl + let init: any + const decl = node.declarationList.declarations[0] + const getVal = () => init ??= solver.solve(decl.initializer!) + if (ts.isIdentifier(decl.name)) { + if (decl.initializer) { + setLazyState(decl.name.text, isExported(node), getVal) + } else { + state.set(decl.name.text, uninitialized) + } + } else { + solveBindingPattern(decl.name, getVal, isExported(node)) + } + + if (isUsing(node)) { + getUsingState().push(lazy(getVal)) + } + } else if (ts.isExpressionStatement(node)) { + solveExpressionStatement(node) + } else if (ts.isReturnStatement(node)) { + const ret = node.expression ? solve(node.expression) : undefined + + return evaluate(ret) + } else if (ts.isImportDeclaration(node)) { + const res = solve(node) + if (res !== undefined) { + for (const [k, v] of Object.entries(res)) { + // FIXME: this doesn't reset the lazy eval + state.set(k, v) + } + } + } else if (ts.isExportDeclaration(node)) { + const res = solveExportDeclaration(node) + if (res !== undefined && state.has('exports')) { + const exports = state.get('exports')! + for (const [k, v] of Object.entries(res)) { + exports[k] = v + } + } + } else if (ts.isIfStatement(node)) { + // BUG: we need to differentiate user-space `undefined` from our own + const res = solveIfStatement(node) + if (res !== undefined) { + return res + } + } else if (ts.isForOfStatement(node)) { + solveForOfStatement(node) + } else if (ts.isForStatement(node)) { + solveForStatement(node) + } else if (ts.isWhileStatement(node)) { + // BUG: we need to differentiate user-space `undefined` from our own + const res = solveWhileStatement(node) + if (res !== undefined) { + return res + } + } else if (ts.isTryStatement(node)) { + const res = solveTryStatement(node) + if (res !== undefined) { + return res + } + } else if (ts.isSwitchStatement(node)) { + solveSwitchStatement(node) + } + } + + function solveBlock(node: ts.Block) { + try { + for (const statement of node.statements) { + const val = solveStatement(statement) + if (val !== undefined) { + return val + } + } + } finally { + solveUsingState() + } + } + + type FunctionLikeDeclaration = ts.ConstructorDeclaration | ts.FunctionDeclaration | ts.MethodDeclaration + type FunctionLikeExpression = ts.ArrowFunction | ts.FunctionExpression + function solveFunctionLikeDeclaration(target: FunctionLikeDeclaration | FunctionLikeExpression) { + if (!target.body) { + return function () { + if (target.type) { + return wrapType(target.type) + } + + return unknown + } + } + + const isGenerator = !!target.asteriskToken + + // `this` refers to execution scopes + return createInternalFunction(function (this: ts.Node[], thisArg: any, ...args: any[]) { + // If we're called _externally_ then we need to shift all of our args to the left + // This will drop our normal `this` value used for execution scope, so we'll assume + // the scope of the solver instead + const actualThisArg = isInternalThis(this) ? thisArg : this + const actualArgs = isInternalThis(this) ? args : [thisArg, ...args] + const actualThis = isInternalThis(this) ? this : scopes + + const scopedState = new Map() + scopedState.set('this', actualThisArg) + if (isGenerator) { + scopedState.set('__kGeneratorState__', []) + } + + function resolveParam(p: ts.ParameterDeclaration, i: number) { + if (p.dotDotDotToken) { + return actualArgs.slice(i) + } + + const init = p.initializer ? solve(p.initializer) : createUnknown() + + return i >= actualArgs.length ? init : actualArgs[i] + } + + target.parameters.forEach((p, i) => { + const resolved = resolveParam(p, i) + // getTerminalLogger().log(getNodeLocation(target), actualArgs) + + if (!ts.isIdentifier(p.name)) { + for (const element of p.name.elements) { + if (ts.isBindingElement(element)) { + if (!ts.isIdentifier(element.name)) { + failOnNode(`Not an identifier: ${p.name.kind}`, element.name) + } + if (isUnknown(resolved)) { + scopedState.set(element.name.text, createUnknown()) + } else { + scopedState.set(element.name.text, resolved[element.name.text]) + } + } else { + failOnNode(`Not an identifier: ${p.name.kind}`, element) + } + } + } else { + scopedState.set(p.name.text, resolved) + } + }) + + const solver = createSolver([...actualThis, target], scopedState, getSubstitute) + const result = solver.solve(target.body!) + + // XXX: eval the entire body even if we don't reference anything directly + // + // We often want the side effects produced by a function call for the purpose + // of permissions analysis. The alternative is to return side-effect producing + // expressions apart of the call convention + for (const [k, v] of scopedState.entries()) { + evaluate(v) + } + + if (isGenerator) { + return scopedState.get('__kGeneratorState__') + } + + return result + }, target.parameters.length, () => target.getText(target.getSourceFile())) + } + + function solveImportDeclaration(node: ts.ImportDeclaration) { + } + + function solveExportDeclaration(node: ts.ExportDeclaration) { + } + + function solveRegularExpressionLiteral(node: ts.RegularExpressionLiteral) { + return node.text + } + + function solveTemplateExpression(node: ts.TemplateExpression) { + const parts: any[] = [] + parts.push(node.head.text) + + for (const span of node.templateSpans) { + const exp = solve(span.expression) + if (isUnknown(exp)) { + return unknown + } + + parts.push(exp) + parts.push(span.literal.text) + } + + function substituteUnion(parts: any[]): any { + const unionIdx = parts.findIndex(isUnion) + if (unionIdx === -1) { + return parts.map(String).join('') + } + + const head = parts.slice(0, unionIdx) + + return createUnion(function* () { + for (const part of parts[unionIdx]) { + if (isUnknown(part)) { + yield unknown + } else { + yield substituteUnion([...head, part, ...parts.slice(unionIdx + 1)]) + } + } + }) + } + + return substituteUnion(parts) + } + + + function solveBinaryExpression(node: ts.BinaryExpression) { + const left = solve(node.left) + const right = solve(node.right) + + if (left === unknown || right === unknown) { + return unknown + } + + // TODO: this only happens from `try/catch` atm + if (left === uninitialized || right === uninitialized) { + return unknown + } + + switch (node.operatorToken.kind) { + case ts.SyntaxKind.BarBarToken: + return left || right + case ts.SyntaxKind.PlusToken: + return left + right + case ts.SyntaxKind.LessThanToken: + return left < right + case ts.SyntaxKind.QuestionQuestionToken: + return left ?? right + } + + return unknown + } + + function solveScopedState(node: ts.Node, target: ts.Node = node) { + const scopedState = new Map() + const solver = createSolver([...scopes, node], scopedState, getSubstitute) + solver.solve(target) + // XXX: eval the entire body even if we don't reference anything directly + for (const [k, v] of scopedState.entries()) { + evaluate(v) + } + } + + // CONTROL FLOW + function solveIfStatement(node: ts.IfStatement) { + const cond = solve(node.expression) + // if (cond === unknownSymbol) { + // return unknownSymbol + // } + + solveScopedState(node, node.thenStatement) + if (node.elseStatement) { + solveScopedState(node, node.elseStatement) + } + } + + function solveSwitchStatement(node: ts.SwitchStatement) { + const exp = solve(node.expression) + for (const clause of node.caseBlock.clauses) { + solveCaseOrDefaultClause(clause) + } + } + + function solveCaseOrDefaultClause(node: ts.CaseOrDefaultClause) { + const scopedState = new Map() + const solver = createSolver([...scopes, node], scopedState, getSubstitute) + + if (ts.isCaseClause(node)) { + const val = solver.solve(node.expression) + } + + for (const statement of node.statements) { + solver.solve(statement) + } + + // XXX: eval the entire body even if we don't reference anything directly + for (const [k, v] of scopedState.entries()) { + evaluate(v) + } + } + + function solveConditionalExpression(node: ts.ConditionalExpression) { + const cond = solve(node.condition) + solve(node.whenTrue) + solve(node.whenFalse) + } + + function solvePrefixUnaryExpression(node: ts.PrefixUnaryExpression) { + const val = solve(node.operand) + if (val === unknown) { + return unknown + } + + switch (node.operator) { + case ts.SyntaxKind.PlusToken: + return +val + case ts.SyntaxKind.MinusToken: + return -val + case ts.SyntaxKind.ExclamationToken: + return !val + } + + failOnNode(`Not implemented: ${node.operator}`, node) + } + + function solveArrayLiteralExpression(node: ts.ArrayLiteralExpression) { + const result: any[] = [] + for (const e of node.elements) { + if (ts.isSpreadElement(e)) { + const val = solve(e.expression) + if (val === unknown) { + return unknown + } else { + if (isUnion(val)) { + // XXX: NOT CORRECT, this should return a union + for (const x of val) { + if (!isUnknown(x)) { + result.push(...x) + } else { + result.push(x) + } + } + } else { + result.push(...val) + } + } + } else { + const val = solve(e) + if (isUnion(val)) { + // XXX: not correct, we should be returning a union + result.push(...val) + } else { + result.push(val) + } + } + } + + return Object.setPrototypeOf(result, _Array.prototype) + } + + function solveTypeofExpression(node: ts.TypeOfExpression) { + const val = solve(node.expression) + if (val === unknown) { + return val + } + + return typeof val + } + + // ERROR HANDLING + function solveTryStatement(node: ts.TryStatement) { + solveScopedState(node, node.tryBlock) + if (node.catchClause) { + // TODO: solve variable decl + solveScopedState(node, node.catchClause.block) + } + if (node.finallyBlock) { + solveScopedState(node, node.finallyBlock) + } + } + + // CONTROL FLOW + function solveWhileStatement(node: ts.WhileStatement) { + const scopedState = new Map() + const solver = createSolver([...scopes, node], scopedState, getSubstitute) + solver.solve(node.statement) + + // TODO: we could simulate evaluation of the loop with iteration and/or time bounds + + // XXX: eval the entire body even if we don't reference anything directly + for (const [k, v] of scopedState.entries()) { + evaluate(v) + } + } + + // CONTROL FLOW + function solveForOfStatement(node: ts.ForOfStatement) { + if (!ts.isVariableDeclarationList(node.initializer)) { + // TODO + return unknown + } + const val = solve(node.expression) + const decl = node.initializer.declarations[0] + if (ts.isIdentifier(decl.name)) { + const scopedState = new Map() + scopedState.set(decl.name.text, val) + // getTerminalLogger().log('iterator', decl.name.text) + const solver = createSolver([...scopes, node], scopedState, getSubstitute) + solver.solve(node.statement) + + // XXX: eval the entire body even if we don't reference anything directly + for (const [k, v] of scopedState.entries()) { + evaluate(v) + } + } + + if (ts.isArrayBindingPattern(decl.name)) { + const scopedState = new Map() + for (let i = 0; i < decl.name.elements.length; i++) { + const n = decl.name.elements[i] + if (ts.isBindingElement(n) && ts.isIdentifier(n.name)) { + if (val === unknown) { + scopedState.set(n.name.text, unknown) + } else { + scopedState.set(n.name.text, createUnion(function* () { + for (const x of val) { + if (isUnknown(x)) { + yield x + } else { + yield x[i] + } + } + })) + } + } + } + + const solver = createSolver([...scopes, node], scopedState, getSubstitute) + solver.solve(node.statement) + + // XXX: eval the entire body even if we don't reference anything directly + for (const [k, v] of scopedState.entries()) { + evaluate(v) + } + } + + return unknown // TODO + } + + function solveForStatement(node: ts.ForStatement) { + if (!node.initializer || !ts.isVariableDeclarationList(node.initializer)) { + // TODO + return unknown + } + const decl = node.initializer.declarations[0] + if (ts.isIdentifier(decl.name) && decl.initializer) { + const scopedState = new Map() + scopedState.set(decl.name.text, solve(decl.initializer)) + // getTerminalLogger().log('iterator', decl.name.text) + const solver = createSolver([...scopes, node], scopedState, getSubstitute) + solver.solve(node.statement) + + // XXX: eval the entire body even if we don't reference anything directly + for (const [k, v] of scopedState.entries()) { + evaluate(v) + } + } + + return unknown // TODO + } + + function solveDeleteExpression(node: ts.DeleteExpression) { + if (ts.isPropertyAccessExpression(node.expression) || ts.isElementAccessExpression(node.expression)) { + const target = solve(node.expression.expression) + if (target === unknown) { + return unknown + } + + if (ts.isPropertyAccessExpression(node.expression)) { + delete target[node.expression.name.text] + } else { + const key = solve(node.expression.argumentExpression) + if (key === unknown) { + return unknown + } + delete target[key] + } + } + + return unknown + } + + function solveYieldExpression(node: ts.YieldExpression) { + if (node.expression) { + const val = solve(node.expression) + const genState = state.get('__kGeneratorState__') // XXX + if (genState) { + genState.push(val) + } + } + } + + function solveSourceFile(node: ts.SourceFile) { + for (const statement of node.statements) { + solveStatement(statement) + } + } + + // TODO: VoidExpression + function solve(node: ts.Node): any { + try { + const val = ts.isImportDeclaration(node) + ? fn() + : substitute?.(node, scopes) ?? fn() + + return val + } catch (e) { + failOnNode(((e as any).message + '\n' + (e as any).stack?.split('\n').slice(1).join('\n')).slice(0, 2048), node) + } + + function fn () { + if (node.kind === ts.SyntaxKind.NullKeyword) { + return null + } else if (node.kind === ts.SyntaxKind.TrueKeyword) { + return true + } else if (node.kind === ts.SyntaxKind.FalseKeyword) { + return false + } else if (ts.isVoidExpression(node)) { + return (solve(node.expression), undefined) // FIXME: use symbol for `undefined` + } else if (node.kind === ts.SyntaxKind.SuperKeyword) { + return solveSuperExpression(node as ts.SuperExpression) + } else if (ts.isStringLiteral(node)) { + return node.text + } else if (ts.isNumericLiteral(node)) { + return Number(node.text) + } else if (ts.isNoSubstitutionTemplateLiteral(node)) { + return node.text + } else if (ts.isArrayLiteralExpression(node)) { + return solveArrayLiteralExpression(node) + } else if (ts.isAwaitExpression(node)) { + return solve(node.expression) + } else if (ts.isAsExpression(node)) { + return solve(node.expression) + } else if (ts.isSatisfiesExpression(node)) { + return solve(node.expression) + } else if (ts.isNonNullExpression(node)) { + return solve(node.expression) + } else if (ts.isParenthesizedExpression(node)) { + return solve(node.expression) + } else if (ts.isYieldExpression(node)) { + return solveYieldExpression(node) + } else if (ts.isDeleteExpression(node)) { + return solveDeleteExpression(node) + } else if (ts.isTypeOfExpression(node)) { + return solveTypeofExpression(node) + } else if (ts.isPrefixUnaryExpression(node)) { + return solvePrefixUnaryExpression(node) + } else if (ts.isRegularExpressionLiteral(node)) { + return solveRegularExpressionLiteral(node) + } else if (ts.isTemplateExpression(node)) { + return solveTemplateExpression(node) + } else if (ts.isImportDeclaration(node)) { + return solveImportDeclaration(node) + } else if (ts.isTaggedTemplateExpression(node)) { + return unknown // XXX + } else if (ts.isObjectLiteralExpression(node)) { + return solveObjectLiteral(node) + } else if (ts.isIdentifier(node)) { + return solveIdentifier(node) + } else if (node.kind === ts.SyntaxKind.ThisKeyword) { + return solveThisKeyword(node) + } else if (ts.isBinaryExpression(node)) { + return solveBinaryExpression(node) + } else if (ts.isPropertyAccessExpression(node)) { + return solvePropertyAccess(node) + } else if (ts.isElementAccessExpression(node)) { + return solveElementAccessExpression(node) + } else if (ts.isClassDeclaration(node)) { + return solveClassDeclarationOrExpression(node) + } else if (ts.isClassExpression(node)) { + return solveClassDeclarationOrExpression(node) + } else if (ts.isCallExpression(node)) { + return solveCallExpression(node) + } else if (ts.isNewExpression(node)) { + return solveNewExpression(node) + } else if (ts.isArrowFunction(node) || ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node) || ts.isConstructorDeclaration(node)) { + return solveFunctionLikeDeclaration(node) + } else if (ts.isFunctionExpression(node)) { + return solveFunctionLikeDeclaration(node) + } else if (ts.isPropertyName(node)) { + return solvePropertyName(node) + } else if (ts.isForOfStatement(node)) { + return solveForOfStatement(node) + } else if (ts.isForStatement(node)) { + return solveForStatement(node) + } else if (ts.isIfStatement(node)) { + return solveIfStatement(node) + } else if (ts.isSwitchStatement(node)) { + return solveSwitchStatement(node) + } else if (ts.isBlock(node)) { + return solveBlock(node) + } else if (ts.isSourceFile(node)) { + return solveSourceFile(node) + } else if (ts.isStatement(node)) { + return solveStatement(node) + } else if (ts.isConditionalExpression(node)) { + return solveConditionalExpression(node) + } + + if (ts.isWhileStatement(node)) { + return + } + + // TODO + if (ts.isJsxElement(node) || ts.isJsxSelfClosingElement(node)) { + return + } + + failOnNode('Not implemented', node) + } + } + + return { solve, getState: () => state, printState } + } + + function findNodeByName(node: ts.Node, name: string): ts.Node | undefined { + if (ts.isIdentifier(node) && node.text === name) { + if (ts.isClassDeclaration(node.parent)) { + return node.parent + } + } + + return node.forEachChild(n => findNodeByName(n, name)) + } + + function createInstance(node: ts.ClassDeclaration, ...args: any[]) { + const solver = createSolver([], undefined) + solver.solve(ts.findAncestor(node, ts.isSourceFile)!) + const ctor = solver.solve(node) + const instance = ctor.call([], {}, ...args) // FIXME: add `node` to scope?? + if (instance === unknown) { + return unknown + } + + return new Proxy(instance, { + get: (target, prop, recv) => { + const val = Reflect.get(target, prop, recv) + if (typeof val === 'function') { + return (...args: any[]) => val.call([], instance, ...args) + } + + return val + } + }) + } + + function invoke(node: ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction, thisArg: any, ...args: any[]) { + const solver = createSolver([], undefined) + solver.solve(ts.findAncestor(node, ts.isSourceFile)!) // FIXME: figure out how to avoid this + + const fn = solver.solve(node) + + return fn.call([node], thisArg, ...args) + } + + return { + ...createSolver(), + createSolver, + createInstance, + invoke, + } +} diff --git a/src/static-solver/utils.ts b/src/static-solver/utils.ts new file mode 100644 index 0000000..3ab5ba5 --- /dev/null +++ b/src/static-solver/utils.ts @@ -0,0 +1,167 @@ +import ts from 'typescript' +import * as path from 'node:path' +import * as fs from 'node:fs' +import { SourceMapV3, toInline, findSourceMap } from '../runtime/sourceMaps' +import { SourceMap } from 'node:module' + +// FIXME: delete this file? + +export function getNodeLocation(node: ts.Node) { + const sf = node.getSourceFile() + if (!sf) { + return + } + + const pos = node.pos + node.getLeadingTriviaWidth(sf) + const lc = sf.getLineAndCharacterOfPosition(pos) + + return `${sf.fileName}:${lc.line + 1}:${lc.character + 1}` +} + +export function failOnNode(message: string, node: ts.Node, showSourceWhenMissing = true): never { + node = ts.getOriginalNode(node) + const location = getNodeLocation(node) + if (!location && showSourceWhenMissing) { + throw new Error(`${message} [kind: ${node.kind}]\n${printNodes([node])}`) + } + + throw new Error(`${message} [kind: ${node.kind}] (${location ?? 'missing source file'})`) +} + + +const createSourcefile = () => ts.factory.createSourceFile([], ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), ts.NodeFlags.None) + +type PrinterOptions = ts.PrinterOptions & { emitSourceMap?: boolean; sourceMapRootDir?: string; inlineSourceMap?: boolean; handlers?: ts.PrintHandlers } + +interface EmitChunk { + readonly text: string + readonly sourcemap?: SourceMapV3 +} + +export function emitChunk(host: SourceMapHost, sourceFile: ts.SourceFile, statements?: ts.Statement[], opt?: PrinterOptions): EmitChunk +export function emitChunk(host: SourceMapHost, sourceFile: ts.SourceFile, statements: ts.Statement[], opt: PrinterOptions & { emitSourceMap: true }): Required +export function emitChunk(host: SourceMapHost, sourceFile: ts.SourceFile, statements?: ts.Statement[], opt: PrinterOptions = {}) { + const updated = statements ? ts.factory.updateSourceFile(sourceFile, statements) : sourceFile + const rootDir = host.getCurrentDirectory() + const generator = opt.emitSourceMap + ? createSourceMapGenerator(host, path.relative(rootDir, sourceFile.fileName), undefined, rootDir, {}) + : undefined + + const writer = createTextWriter() + const handlers = opt.handlers + delete opt.handlers + + const printer = ts.createPrinter({ inlineSourceMap: true, ...opt } as any, handlers) as ts.Printer & { + writeFile: (sourceFile: ts.SourceFile, writer: TextWriter, generator?: SourceMapGenerator) => void + } + + printer.writeFile(updated, writer, generator) + + return { + text: writer.getText(), + sourcemap: generator?.toJSON(), + } +} + +// return + '\n\n' + toInline(sourcemap) + +export function printNodes(nodes: ts.Node[], source = nodes[0]?.getSourceFile() ?? createSourcefile(), options?: PrinterOptions) { + const printer = ts.createPrinter(options) + const result = nodes.map(n => printer.printNode(ts.EmitHint.Unspecified, n, source)) + + return result.join('\n') +} + +export function isNonNullable(val: U): val is NonNullable { + return val !== undefined && val !== null +} + +export function createVariableStatement(name: string | ts.Identifier, exp: ts.Expression, modifiers?: ts.ModifierSyntaxKind[], factory = ts.factory) { + const modifierTokens = modifiers?.map(m => factory.createModifier(m)) + const decl = factory.createVariableDeclaration(name, undefined, undefined, exp) + + return factory.createVariableStatement( + modifierTokens, + factory.createVariableDeclarationList([decl], ts.NodeFlags.Const) + ) +} + +interface ObjectLiteralInput { + readonly [key: string]: Literal | Literal[] +} + +type Literal = string | number | boolean | ts.Expression | ObjectLiteralInput | undefined + +export function getModuleSpecifier(node: ts.ImportDeclaration | ts.ExportDeclaration): ts.StringLiteral { + if (!node.moduleSpecifier || !ts.isStringLiteral(node.moduleSpecifier)) { + failOnNode('No module specifier found', node) + } + + return node.moduleSpecifier +} + +function getUtil(): typeof import('node:util') { + return require('node:util') +} + +export function getNullTransformationContext(): ts.TransformationContext { + if (!('nullTransformationContext' in ts)) { + const inspected = getUtil().inspect(ts, { + colors: false, + showHidden: true, + compact: true, + depth: 1, + sorted: true, + }) + + throw new Error(`No transformation context found. Current exports: ${inspected}`) + } + + return (ts as any).nullTransformationContext +} + +// Not correct +interface TextWriter { + write(str: string): void + getLine(): number + getColumn(): number + getText(): string +} + +function createTextWriter(): TextWriter { + if (!('createTextWriter' in ts)) { + throw new Error(`No text writer found`) + } + + return (ts as any).createTextWriter('\n') +} + +// Not correct +interface SourceMapGenerator { + addMapping(...args: unknown[]): unknown + addSource(fileName: string): number + toJSON(): SourceMapV3 +} + +export interface SourceMapHost { + getCurrentDirectory(): string + getCanonicalFileName: (fileName: string) => string +} + +function createSourceMapGenerator(host: SourceMapHost, fileName: string, sourceRoot: string | undefined, sourcesDirectoryPath: string, opt: any): SourceMapGenerator { + if (!('createSourceMapGenerator' in ts)) { + throw new Error(`No source map generator found`) + } + + return (ts as any).createSourceMapGenerator(host, fileName, sourceRoot, sourcesDirectoryPath, opt) +} + +export function extract(arr: T[], index: number): T { + const v = arr[index] + arr[index] = arr[arr.length - 1] + arr.length -= 1 + + return v +} + + diff --git a/src/system.ts b/src/system.ts new file mode 100644 index 0000000..b110add --- /dev/null +++ b/src/system.ts @@ -0,0 +1,448 @@ +import * as fs from 'node:fs' +import * as path from 'node:path' + +export interface FsEntity { + readonly type: 'file' | 'directory' | 'unknown' + readonly name: string +} + +export interface FsEntityStats { + readonly type: 'file' | 'directory' | 'unknown' + readonly size: number + readonly mtimeMs: number + readonly ctimeMs: number + + readonly hash?: string +} + +export type WriteFileOptions = BufferEncoding | { + readonly mode?: fs.Mode + readonly encoding?: BufferEncoding + readonly flag?: 'w' | 'wx' // | 'w+' | 'wx+' + + /** Using `#mem` writes a temporary in-memory file */ + readonly fsKey?: '#mem' | string + readonly checkChanged?: boolean +} + +interface LinkOptions { + readonly overwrite?: boolean + readonly symbolic?: boolean + readonly typeHint?: 'dir' | 'file' | 'junction' + readonly mode?: fs.Mode +} + +export interface Fs { + readDirectory(fileName: string): Promise + writeFile(fileName: string, data: Uint8Array): Promise + writeFile(fileName: string, data: string, encoding?: BufferEncoding): Promise + writeFile(fileName: string, data: Uint8Array | string, encoding?: BufferEncoding): Promise + writeFile(fileName: string, data: Uint8Array | string, options?: WriteFileOptions): Promise + readFile(fileName: string): Promise + readFile(fileName: string, encoding: BufferEncoding): Promise + deleteFile(fileName: string, opt?: { recursive?: boolean; force?: boolean; fsKey?: '#mem' }): Promise + fileExists(fileName: string): Promise + // open(fileName: string): Promise + link(existingPath: string, newPath: string, opt?: LinkOptions): Promise + stat(fileName: string): Promise +} + +export interface JsonFs { + writeJson(fileName: string, data: any): Promise + readJson(fileName: string): Promise +} + +export interface SyncFs { + writeFileSync(fileName: string, data: Uint8Array): void + writeFileSync(fileName: string, data: string, encoding?: BufferEncoding): void + writeFileSync(fileName: string, data: Uint8Array | string, encoding?: BufferEncoding): void + writeFileSync(fileName: string, data: Uint8Array | string, options?: WriteFileOptions): void + readFileSync(fileName: string): Uint8Array + readFileSync(fileName: string, encoding: BufferEncoding): string + // readFileSync(fileName: string, encoding?: BufferEncoding): string | Buffer + deleteFileSync(fileName: string): void + fileExistsSync(fileName: string): boolean + // linkSync(existingPath: string, newPath: string): void +} + +export interface FileHandle { + // readonly position: number + write(data: Uint8Array): Promise + write(data: string, encoding?: BufferEncoding): Promise + write(data: Uint8Array | string, encoding?: BufferEncoding): Promise + // read(length: number): Promise + // read(length: number, encoding: BufferEncoding): Promise + dispose(): Promise +} + +// Note: +// On Linux, positional writes don't work when the file is opened in append mode. The kernel ignores +// the position argument and always appends the data to the end of the file. + +// readFile: (position: number, buffer: Uint8Array, length: string, offset?: number) => Promise + +type WriteFileFn = (position: number, buffer: Uint8Array, length: number, offset?: number) => Promise + +function createFileHandle(writeFile: WriteFileFn, close: () => Promise): FileHandle { + let writePosition = 0 + let isWorking = false + let closingCallback: (err?: any) => void | undefined + const queuedWrites: { buffer: Uint8Array, callback: (err?: any) => void }[] = [] + + async function doWork() { + while (queuedWrites.length > 0) { + const task = queuedWrites.shift()! + try { + await writeFile(writePosition, task.buffer, task.buffer.length) + writePosition += task.buffer.length + task.callback() + } catch (e) { + task.callback(e) + + // All remaining tasks must be evicted + const tasks = [...queuedWrites] + queuedWrites.length = 0 + tasks.forEach(t => t.callback(e)) + closingCallback?.(e) + } + + } + + isWorking = false + closingCallback?.() + } + + function write(data: Uint8Array | string, encoding?: BufferEncoding) { + if (closingCallback !== undefined) { + throw new Error(`Cannot write to a disposed file handle`) + } + + const buffer = typeof data === 'string' ? Buffer.from(data, encoding) : data + + return new Promise((resolve, reject) => { + const callback = (err?: any) => err ? reject(err) : resolve() + queuedWrites.push({ buffer, callback }) + if (!isWorking) { + isWorking = true + doWork() + } + }) + } + + function dispose() { + const flushed = new Promise((resolve, reject) => { + closingCallback = err => err ? reject(err) : resolve() + }) + + const lastTask = queuedWrites[queuedWrites.length - 1] + if (!lastTask) { + return close() + } + + return flushed.finally(close) + } + + return { write, dispose } +} + +export function watchForFile(fileName: string) { + const watcher = fs.watch(path.dirname(fileName)) + + function dispose() { + watcher.close() + } + + function onFile(listener: () => void) { + watcher.on('change', ev => { + if (ev === path.basename(fileName)) { + listener() + } + }) + } + + return { onFile, dispose } +} + +export async function openHandle(fileName: string): Promise { + await fs.promises.mkdir(path.dirname(fileName), { recursive: true }) + + const fd = await new Promise((resolve, reject) => { + fs.open(fileName, 'w', (err, fd) => err ? reject(err) : resolve(fd)) + }) + + const writeFile: WriteFileFn = (position, buffer, length, offset) => { + return new Promise((resolve, reject) => { + fs.write(fd, buffer, offset, length, position, err => err ? reject(err) : resolve()) + }) + } + + const closeFile = () => { + return new Promise((resolve, reject) => { + fs.close(fd, err => err ? reject(err) : resolve()) + }) + } + + return createFileHandle(writeFile, closeFile) +} + +class FsError extends Error { + constructor(message: string, public readonly code: string) { + super(message) + } +} + +export async function ensureDir(fileName: string) { + // TODO: delete file if it's a sym link ? + await fs.promises.mkdir(path.dirname(fileName), { recursive: true }) +} + +export function ensureDirSync(fileName: string) { + fs.mkdirSync(path.dirname(fileName), { recursive: true }) +} + +function mapType(f: fs.Dirent | fs.Stats): FsEntityStats['type'] { + return f.isDirectory() ? 'directory' : f.isFile() ? 'file' : 'unknown' +} + +export function readDirectorySync(dir: string) { + const res: FsEntity[] = [] + for (const f of fs.readdirSync(dir, { withFileTypes: true })) { + res.push({ type: mapType(f), name: f.name }) + } + + return res +} + +export async function readDirRecursive(fs: Fs, dir: string) { + const files: Record = {} + async function readDir(filePath: string) { + for (const f of await fs.readDirectory(filePath)) { + const absPath = path.resolve(filePath, f.name) + if (f.type === 'directory') { + await readDir(absPath) + } else if (f.type === 'file') { + files[path.relative(dir, absPath)] = absPath + } + } + } + + await readDir(dir) + + return files +} + +function isTooManyFilesError(err: unknown): err is Error & { code: 'EMFILE' } { + return (err as any).code === 'EMFILE' +} + +export function createLocalFs(): Fs & SyncFs { + const memory = new Map() + + // const queue: [resolve: () => void, reject: (err: any) => void, fileName: string, data: string | Uint8Array, opt?: WriteFileOptions][] = [] + + async function writeFile(fileName: string, data: string | Uint8Array, opt?: WriteFileOptions) { + if (typeof opt !== 'string' && opt?.fsKey === '#mem') { + if (opt.flag === 'wx' && memory.has(fileName)) { + throw new FsError(`File "${fileName}" already exists in memory`, 'EEXIST') + } + + memory.set(fileName, { data, encoding: opt.encoding ?? (typeof data === 'string' ? 'utf-8' : undefined) }) + + return + } + + try { + await fs.promises.writeFile(fileName, data, opt) + } catch (e) { + if (isTooManyFilesError(e)) { + // XXX: not the best impl. + return new Promise((resolve, reject) => { + setTimeout( + () => writeFile(fileName, data, opt).then(resolve, reject), + 100 + (250 * Math.random()) + ) + }) + } + + if ((e as any).code !== 'ENOENT') { // TODO: throw when using 'append' (or equivalent) flags + throw e + } + + await ensureDir(fileName) + try { + await fs.promises.writeFile(fileName, data, opt) + } catch (e) { + if (isTooManyFilesError(e)) { + // XXX: not the best impl. + return new Promise((resolve, reject) => { + setTimeout( + () => writeFile(fileName, data, opt).then(resolve, reject), + 100 + (250 * Math.random()) + ) + }) + } + throw e + } + } + } + + function writeFileSync(fileName: string, data: string | Uint8Array, opt?: WriteFileOptions) { + ensureDirSync(fileName) + fs.writeFileSync(fileName, data, opt) + } + + async function deleteFile(fileName: string, opt?: { recursive?: boolean; force?: boolean; fsKey?: '#mem' }) { + if (memory.has(fileName)) { + memory.delete(fileName) + return + } else if (opt?.fsKey === '#mem') { + return + } + + const stat = await fs.promises.lstat(fileName) + if (stat.isDirectory()) { + if (opt && !opt.force) { + await fs.promises.rmdir(fileName, opt) + } else { + await fs.promises.rm(fileName, opt ?? { recursive: true, force: true }) + } + } else if (stat.isSymbolicLink()) { + await fs.promises.unlink(fileName) + } else { + await fs.promises.rm(fileName) // TODO: use flag for `force` ? + } + } + + function decode(data: Uint8Array, encoding: BufferEncoding) { + if (Buffer.isBuffer(data)) { + return data.toString(encoding) + } + + return Buffer.from(data).toString(encoding) + } + + async function readFile(fileName: string, encoding?: BufferEncoding) { + if (memory.has(fileName)) { + const r = memory.get(fileName)! + if (r.encoding === encoding) { + return r.data + } + + const encoded = typeof r.data === 'string' ? Buffer.from(r.data, r.encoding) : r.data + memory.set(fileName, { data: encoded }) + + return encoding ? decode(encoded, encoding) : encoded + } + + try { + return fs.promises.readFile(fileName, encoding).catch(e => { + const err = new Error((e as any).message) + throw Object.assign(e, { get stack() { return err.stack } }) + }) as any + } catch (e) { + if (isTooManyFilesError(e)) { + // XXX: not the best impl. + return new Promise((resolve, reject) => { + setTimeout( + () => readFile(fileName, encoding).then(resolve, reject), + 100 + (250 * Math.random()) + ) + }) + } + throw e + } + } + + async function fileExists(fileName: string) { + return fs.promises.access(fileName, fs.constants.F_OK).then(() => true, () => false) + } + + async function stat(fileName: string) { + const stats = await fs.promises.stat(fileName).catch(e => { + const err = new Error((e as any).message) + throw Object.assign(e, { get stack() { return err.stack } }) + }) + + return { + type: mapType(stats), + size: stats.size, + mtimeMs: stats.mtimeMs, + ctimeMs: stats.ctimeMs, + } + } + + async function readDirectory(fileName: string) { + const res: FsEntity[] = [] + for (const f of await fs.promises.readdir(fileName, { withFileTypes: true })) { + res.push({ type: mapType(f), name: f.name }) + } + + return res + } + + return { + stat, + // open, + writeFile, + writeFileSync, + link: async (existingPath, newPath, opt) => { + async function _link() { + if (opt?.symbolic) { + await fs.promises.symlink(existingPath, newPath, opt.typeHint) + } else { + await fs.promises.link(existingPath, newPath) + } + + if (opt?.mode) { + await fs.promises.chmod(newPath, opt.mode) + } + } + + try { + await _link() + } catch (e) { + if ((e as any).code === 'EEXIST') { + if (opt?.overwrite === false) { + return + } + + await fs.promises.unlink(newPath).catch(async e => { + if ((e as any).code === 'EPERM' && process.platform === 'win32') { + // This isn't a symlink + const realpath = await fs.promises.realpath(newPath) + if (realpath === newPath) { + try { + return await fs.promises.rm(newPath) + } catch (e) { + // ok, I guess `rm` doesn't behave the same on Windows + if ((e as any).code === 'ERR_FS_EISDIR' || (e as any).code === 'EISDIR') { + return fs.promises.rmdir(newPath) + } + } + } + } + + throw e + }) + return _link() + } + + if ((e as any).code !== 'ENOENT') { + throw e + } + + await ensureDir(newPath) + + return _link() + } + }, + deleteFile, + deleteFileSync: (fileName) => fs.rmSync(fileName), + readFile, + readFileSync: (fileName, encoding?: BufferEncoding) => fs.readFileSync(fileName, encoding) as any, + fileExists, + fileExistsSync: fs.existsSync, + readDirectory, + } +} + diff --git a/src/templates.ts b/src/templates.ts new file mode 100644 index 0000000..9a05e9c --- /dev/null +++ b/src/templates.ts @@ -0,0 +1,174 @@ +import * as path from 'node:path' +import { TfJson } from './runtime/modules/terraform' +import * as utils from './utils' +import { getLogger } from './logging' +import { getDeploymentBuildDirectory, getRootDir } from './workspaces' +import { getFsFromHash, getDeploymentFs, getProgramFs, getTemplate } from './artifacts' +import { getBuildTargetOrThrow, getFs } from './execution' +import { getSecret } from './services/secrets' + +// Structures for modeling interations between our "local" +// template (the goal state) and the "remote" state (the current state) + +const builtinResourceType = 'synapse_resource' + +interface BaseConfig { + readonly module_name: string +} + +interface ResourceIdentifier { + readonly type: string + readonly name: string +} + +export interface ConfigResource extends ResourceIdentifier { + readonly mode: 'managed' | 'data' + readonly fileName: string + readonly testSuiteId?: number + readonly config: T +} + +export interface BuiltinResource extends ConfigResource> { + readonly type: typeof builtinResourceType +} + +export interface BuiltinResourceConfig extends BaseConfig { + readonly type: string + readonly input: T +} + +export interface Resource extends ConfigResource { + readonly state: U +} + +export interface OrphanResource extends ResourceIdentifier { + readonly state: U +} + +export function parseModuleName(moduleName: string) { + const [fileName, rem] = moduleName.split('#') + const params = new URLSearchParams(rem) + const testSuiteId = Number(params.get('test-suite') ?? undefined) // Ensure `null` is treated as `NaN` + + return { + fileName, + testSuiteId: isNaN(testSuiteId) ? undefined : testSuiteId, + } +} + +export function listConfigResources(template: TfJson) { + const resources: ConfigResource[] = [] + + function gather(segment: Record>, mode: ConfigResource['mode']) { + for (const [type, group] of Object.entries(segment)) { + for (const [name, config] of Object.entries(group)) { + resources.push({ + mode, + name, + type, + config, + ...parseModuleName(config.module_name), + }) + } + } + } + + gather(template.data, 'data') + gather(template.resource, 'managed') + + return resources +} + +export function getHash(template: TfJson) { + return utils.getHash(JSON.stringify({ ...template, '//': undefined }), 'base64url') +} + +export type QualifiedId = `${string}.${string}` +export function getId(resource: ResourceIdentifier): QualifiedId { + return `${resource.type}.${resource.name}` +} + +export type TemplateService = ReturnType +export function createTemplateService( + fs = getFs(), + programFsHash?: string, +) { + let templateHash: string | undefined + + async function _getTemplateFilePath() { + const bt = getBuildTargetOrThrow() + const dest = path.resolve(getDeploymentBuildDirectory(bt), 'stack.tf.json') + const programFs = programFsHash ? await getFsFromHash(programFsHash) : getProgramFs() + const template = await getTemplate(programFs) + if (!template) { + throw new Error(`No template found`) + } + + templateHash = getHash(template) + await fs.writeFile(dest, JSON.stringify(template)) + + return dest + } + + const getTemplateFilePath = utils.memoize(_getTemplateFilePath) + + async function getTemplate2(): Promise { + const templateFilePath = await getTemplateFilePath() + + return JSON.parse(await fs.readFile(templateFilePath, 'utf-8')) + } + + async function setTemplate(template: TfJson): Promise { + templateHash = getHash(template) + + // await writeTemplate(getProgramFs(), template) + + const bt = getBuildTargetOrThrow() + const dest = path.resolve(getDeploymentBuildDirectory(bt), 'stack.tf.json') + + await fs.writeFile(dest, JSON.stringify(template)) + } + + async function getTemplateHash() { + return templateHash ??= getHash(await getTemplate2()) + } + + async function tryGetTemplate(): Promise { + try { + return await getTemplate2() + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } + } + + async function getSecretsMap() { + const template = await getTemplate2() + const secrets = template['//']?.secrets + + return secrets + } + + async function getSecretBindings() { + const secrets = await getSecretsMap() + if (!secrets) { + return + } + + const bindings: Record = Object.fromEntries( + await Promise.all(Object.entries(secrets).map(async ([k, v]) => [k, await getSecret(v)])) + ) + + return bindings + } + + return { + getTemplate: getTemplate2, + setTemplate, + tryGetTemplate, + getTemplateFilePath, + getSecretBindings, + getTemplateHash, + } +} \ No newline at end of file diff --git a/src/testing/index.ts b/src/testing/index.ts new file mode 100644 index 0000000..ff3278b --- /dev/null +++ b/src/testing/index.ts @@ -0,0 +1,263 @@ +import type { TestSuite, Test } from '../runtime/modules/test' +import { Logger, getLogger } from '../logging' +import { TemplateService, parseModuleName } from '../templates' +import { readResourceState } from '../artifacts' +import { SessionContext } from '../deploy/deployment' +import { isDataPointer } from '../build-fs/pointers' +import { TfState } from '../deploy/state' +import { keyedMemoize } from '../utils' + +export function createTestRunner( + loader: ReturnType, +) { + function emitStatus(info: TestItem, status: 'running' | 'passed' | 'pending'): void + function emitStatus(info: TestItem, status: 'failed', error: Error): void + function emitStatus(info: TestItem, status: 'running' | 'passed' | 'failed' | 'pending', reason?: Error) { + if (info.hidden) return + + getLogger().emitTestEvent({ + reason, + status, + id: info.id, + name: info.name, + itemType: info.type, + parentId: info.parentId, + } as Parameters[0]) + } + + async function runTest(test: ResolvedTest) { + emitStatus(test, 'running') + + try { + await test.resolved.run() + emitStatus(test, 'passed') + } catch (e) { + const err = e as Error + emitStatus(test, 'failed', err) + + return err + } + } + + async function runSuite(suite: ResolvedSuite) { + emitStatus(suite, 'running') + + const items: TestItem[] = [ + ...suite.tests, + ...suite.suites, + ].sort((a, b) => a.id - b.id) + + const results: { id: number; error: Error }[] = [] + for (const item of items) { + const error = item.type === 'test' ? await runTest(item) : await runSuite(item) + if (error !== undefined) { + results.push({ id: item.id, error }) + } + } + + const errors = results.map(r => r.error) + if (errors.length === 0) { + return emitStatus(suite, 'passed') + } + + const err = new AggregateError(errors, `Test suite failed`) + emitStatus(suite, 'failed', err) + + return err + } + + async function runTestItems(items: TestItem[]) { + const queue: TestItem[] = [] + for (const item of items) { + emitStatus(item, 'pending') + queue.push(item) + } + + while (queue.length > 0) { + const item = queue.shift()! + if (item.type === 'suite') { + await runSuite(item) + } else { + await runTest(item) + } + } + } + + async function loadSuite(location: string) { + const { suite } = await loader.loadModule(location) + if (!suite) { + throw new Error(`Missing suite export`) + } + + if (typeof suite.run !== 'function') { + throw new Error(`Suite object is missing a "run" function`) + } + + return suite as TestSuite + } + + async function resolveSuite(id: string) { + const r = await readResourceState(id.split('.')[1]) + const handler = r.handler + if (!isDataPointer(handler) && typeof handler !== 'string') { + throw new Error(`Expected resource "${id}" to have a handler of type "string", got "${typeof handler}"`) + } + + return loadSuite(handler) + } + + async function loadTestSuites(suites: Record, tests: Record) { + async function _getSuiteWithTests(k: string): Promise { + const v = suites[k] + const resolved = await resolveSuite(k) + const suiteTests = Object.values(tests) + .filter(v => v.parentId === resolved.id) + .map(async info => { + const index = resolved.tests.findIndex(t => t.id === info.id) + if (index === -1) { + throw new Error(`Test not found in suite "${k}": ${info.id}`) + } + + return { + type: 'test', + ...info, + resolved: resolved.tests[index], + } satisfies ResolvedTest + }) + + const childrenSuites = Object.entries(suites) + .filter(([_, v]) => v.parentId === resolved.id) + .map(([k]) => getSuiteWithTests(k)) + + return { + ...v, + type: 'suite', + resolved, + tests: await Promise.all(suiteTests), + suites: await Promise.all(childrenSuites), + } + } + + const getSuiteWithTests = keyedMemoize(_getSuiteWithTests) + + const mapped = Object.entries(suites).map(async ([k, v]) => { + return [k, await getSuiteWithTests(k)] as const + }) + + return Object.fromEntries(await Promise.all(mapped)) + } + + return { + runTestItems, + loadTestSuites, + } +} + +interface BaseTestItem { + id: number + name: string + fileName: string + parentId?: number + hidden?: boolean +} + +interface ResolvedTest extends BaseTestItem { + type: 'test' + resolved: Test +} + +interface ResolvedSuite extends BaseTestItem { + type: 'suite' + resolved: TestSuite + tests: ResolvedTest[] + suites: ResolvedSuite[] +} + +type TestItem = ResolvedTest | ResolvedSuite + +interface TestFilter { + parentId?: number + fileNames?: string[] + targetIds?: number[] +} + +function canIncludeItem(item: BaseTestItem, filter: TestFilter) { + if (filter.parentId !== undefined && item.parentId !== filter.parentId) { + return false + } + + if (filter.fileNames !== undefined && !filter.fileNames.includes(item.fileName)) { + return false + } + + if (filter.targetIds !== undefined && !filter.targetIds.includes(item.id)) { + return false + } + + return true +} + +const csResourceType = 'synapse_resource' +export async function listTests(templates: TemplateService, filter: TestFilter = {}) { + const resources = (await templates.getTemplate()).resource + const csResources: Record = resources?.[csResourceType] ?? {} + + const tests: Record = {} + for (const [k, v] of Object.entries(csResources)) { + if (v.type === 'Test') { + const { fileName, testSuiteId } = parseModuleName(v.module_name) + if (testSuiteId === undefined) { + throw new Error(`Test is missing a suite id: ${k}`) + } + + const name = v.input?.name + if (typeof name !== 'string') { + throw new Error(`Test has no name: ${k}`) + } + + const id = v.input?.id + if (typeof id !== 'number') { + throw new Error(`Test has no id: ${k}`) + } + + const item = { id, name, fileName, parentId: testSuiteId } + if (!canIncludeItem(item, filter)) { + continue + } + + tests[`${csResourceType}.${k}`] = item + } + } + + return tests +} + +export async function listTestSuites(templates: TemplateService, filter: TestFilter = {}) { + const resources = (await templates.getTemplate()).resource + const csResources: Record = resources?.[csResourceType] ?? {} + + const suites: Record = {} + for (const [k, v] of Object.entries(csResources)) { + if (v.type === 'TestSuite') { + const id: number = v.input.id + const { fileName, testSuiteId } = parseModuleName(v.module_name) + const parentId = testSuiteId !== id ? testSuiteId : undefined + + const name = v.input?.name + if (typeof name !== 'string') { + throw new Error(`Test has no name: ${k}`) + } + + const hidden = parentId === undefined + const item : BaseTestItem = { id, name, fileName, parentId, hidden } + if (!canIncludeItem(item, filter)) { + continue + } + + suites[`${csResourceType}.${k}`] = item + } + } + + return suites +} + diff --git a/src/testing/internal.ts b/src/testing/internal.ts new file mode 100644 index 0000000..04277b7 --- /dev/null +++ b/src/testing/internal.ts @@ -0,0 +1,62 @@ +import * as path from 'node:path' +import { getFs, isSelfSea } from '../execution' +import { createNpmLikeCommandRunner } from '../pm/publish' + +// Logic for an internal test runner + +// Commands are treated like scripts in `package.json` +const commandsDirective = '!commands' + +function parseCommands(text: string, synapseCmd = isSelfSea() ? undefined : 'syn') { + const lines = text.split('\n') + const directive = lines.findIndex(l => l.startsWith(`// ${commandsDirective}`)) + if (directive === -1) { + return + } + + const commands: string[] = [] + for (let i = directive + 1; i < lines.length; i++) { + const line = lines[i] + if (!line.startsWith('//')) break + + const [command, ...rest] = line.slice(2).split('#') + const comment = rest.join('#') + + const trimmed = command.trim() + if (trimmed) { + commands.push(trimmed) + } + } + + // Used to support partial dev builds + if (synapseCmd) { + return commands.map(cmd => cmd.replaceAll('synapse', synapseCmd)) + } + + return commands +} + +interface RunTestOptions { + baseline?: boolean + snapshot?: boolean +} + +export async function runInternalTestFile(fileName: string, opt?: RunTestOptions) { + const text = await getFs().readFile(fileName, 'utf-8') + const commands = parseCommands(text) + if (!commands || commands.length === 0) { + throw new Error(`No commands found in test file: ${fileName}`) + } + + if (opt?.snapshot) { + const runner = createNpmLikeCommandRunner(path.dirname(fileName), undefined, ['inherit', 'pipe', 'inherit']) + const cmd = commands.join(' && ') + const result = await runner(cmd) + return + } + + const runner = createNpmLikeCommandRunner(path.dirname(fileName), undefined, 'inherit') + const cmd = commands.join(' && ') + + await runner(cmd) +} diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..ed0f00c --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,2061 @@ +import ts from 'typescript' +import * as path from 'node:path' +import * as assert from 'node:assert' +import * as zlib from 'node:zlib' +import * as fs from 'node:fs/promises' +import * as child_process from 'node:child_process' +import * as util from 'node:util' +import * as crypto from 'node:crypto' +// import type { CompiledChunk } from './artifacts' +import { SourceMapHost, emitChunk } from './static-solver/utils' +import { SourceMapV3 } from './runtime/sourceMaps' +import type { Fs, SyncFs } from './system' +import { homedir } from 'node:os' + +interface ObjectLiteralInput { + [key: string]: Literal | Literal[] +} + +type Literal = boolean | string | number | ts.Expression | ObjectLiteralInput | undefined +type NonNullableLiteral = NonNullable | Literal[] + +export function isNode(obj: unknown): obj is ts.Node { + return ( + !!obj && typeof obj === 'object' && + typeof (obj as any).kind === 'number' && + typeof (obj as any).flags === 'number' + ) +} + +const dataPointer = Symbol.for('synapse.pointer') +function renderObject(o: any, factory = ts.factory): ts.Expression { + if (o === null) { + return factory.createNull() + } + + if (Array.isArray(o)) { + return createArrayLiteral(factory, o) + } + + if (o[dataPointer]) { + return factory.createStringLiteral(o.ref, true) + } + + if (!isNode(o)) { + return createObjectLiteral(o, factory) + } + + return o as ts.Expression +} + +export function createLiteral(v: Literal | Literal[], factory = ts.factory): ts.Expression { + switch (typeof v) { + case 'undefined': + return factory.createIdentifier('undefined') + case 'string': + return factory.createStringLiteral(v, true) + case 'number': + if (v < 0) { + return factory.createPrefixUnaryExpression( + ts.SyntaxKind.MinusToken, + factory.createNumericLiteral(-v), + ) + } + return factory.createNumericLiteral(v) + case 'boolean': + return v ? factory.createTrue() : factory.createFalse() + case 'object': + return renderObject(v) + } + + throw new Error(`invalid value: ${v}`) +} + +export function createArrayLiteral(factory: ts.NodeFactory, arr: Literal[]): ts.ArrayLiteralExpression { + const elements: ts.Expression[] = [] + for (let i = 0; i < arr.length; i++) { + const e = arr[i] + if (e === undefined) { + elements.push(factory.createOmittedExpression()) + } else { + elements.push(createLiteral(e, factory)) + } + } + + return factory.createArrayLiteralExpression(elements, true) +} + +export function createObjectLiteral(obj: ObjectLiteralInput, factory = ts.factory): ts.ObjectLiteralExpression { + const assignments = Object.entries(obj).map(([k, v]) => { + if (typeof v === 'undefined') return + // TODO: only use string literal key if the ident contains invalid characters + // const key = factory.createIdentifier(k) + const key = factory.createStringLiteral(k) + const val = createLiteral(v, factory) + + return factory.createPropertyAssignment(key, val) + }).filter(isNonNullable) + + return factory.createObjectLiteralExpression(assignments, true) +} + +export function createPropertyAssignment(factory: ts.NodeFactory, key: string, value: NonNullableLiteral) { + // TODO: only use string literal key if the ident contains invalid characters + // const key = factory.createIdentifier(k) + + return factory.createPropertyAssignment( + factory.createStringLiteral(key), + createLiteral(value, factory) + ) +} + +export function createConstAssignment(factory: ts.NodeFactory, name: string, initializer: ts.Expression) { + return factory.createVariableStatement( + undefined, + factory.createVariableDeclarationList( + [factory.createVariableDeclaration( + factory.createIdentifier(name), + undefined, + undefined, + initializer + )], + ts.NodeFlags.Const + ) + ) +} + +export function prependStatements(factory: ts.NodeFactory, sourceFile: ts.SourceFile, ...statements: ts.Statement[]) { + return factory.updateSourceFile(sourceFile, [ + ...statements, + ...sourceFile.statements, + ]) +} + +export function createEnvVarAccess(factory: ts.NodeFactory, key: string | ts.Expression) { + return factory.createElementAccessExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier("process"), + factory.createIdentifier("env") + ), + typeof key === 'string' ? factory.createStringLiteral(key) : key + ) +} + +export function reduceAccessExpressionOld(factory: ts.NodeFactory, node: ts.Expression): ts.Expression[] { + if (ts.isElementAccessExpression(node)) { + return [ + ...reduceAccessExpressionOld(factory, node.expression), + node.argumentExpression + ] + } else if (ts.isPropertyAccessExpression(node)) { + return [ + ...reduceAccessExpressionOld(factory, node.expression), + factory.createStringLiteral(node.name.text) + ] + } else if (ts.isNonNullExpression(node)) { + return reduceAccessExpressionOld(factory, node.expression) + } + + return [node] +} + +// doesn't reduce into string lits +export function splitExpression(node: ts.Expression): ts.Expression[] { + if (ts.isElementAccessExpression(node)) { + return [ + ...splitExpression(node.expression), + node.argumentExpression + ] + } else if (ts.isPropertyAccessExpression(node)) { + return [ + ...splitExpression(node.expression), + node.name + ] + } else if (ts.isNonNullExpression(node)) { + return splitExpression(node.expression) + } else if (ts.isParenthesizedExpression(node)) { + return splitExpression(node.expression) + } + + return [node] +} + +export function isNonNullable(val: U): val is NonNullable { + return val !== undefined && val !== null +} + +export function getImportSymbol(node: ts.Node, importDecl: ts.ImportDeclaration, typeChecker: ts.TypeChecker) { + const sym = typeChecker.getSymbolAtLocation(node) + if (!sym) { + return + } + assert.ok(importDecl.importClause, 'Missing import clause') + const name = importDecl.importClause.name + if (name) { + const nameSym = typeChecker.getSymbolAtLocation(name) + if (sym === nameSym) { + return nameSym + } + } + const bindings = importDecl.importClause.namedBindings + if (bindings) { + if (ts.isNamedImports(bindings)) { + for (const binding of bindings.elements) { + const id = binding.propertyName ?? binding.name + const bindingSym = typeChecker.getSymbolAtLocation(id) + if (sym === bindingSym) { + return bindingSym + } + } + } else { + const bindingSym = typeChecker.getSymbolAtLocation(bindings.name) + if (sym === bindingSym) { + return bindingSym + } + } + } +} + +export function parseFqnOfSymbol(sym: ts.Symbol, typeChecker: ts.TypeChecker) { + const fqn = typeChecker.getFullyQualifiedName(sym) + const match = fqn.match(/(?:"(?.+?)")?(?:\.(?.+))?/) + const { name, module } = match?.groups ?? {} + + if (!name && !module) { + if (sym.valueDeclaration || sym.declarations) { + // failOnNode(`Failed to match FQN of symbol (${fqn})`, sym.valueDeclaration) + return { name: fqn, module: undefined } + } else { + throw new Error( `Failed to match FQN: ${fqn}`) + } + } + + return { name, module } +} + +const createSourcefile = () => ts.factory.createSourceFile([], ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), ts.NodeFlags.None) + +export function printNodes(nodes: readonly ts.Node[], source = nodes[0].getSourceFile() ?? createSourcefile(), options?: ts.PrinterOptions) { + const printer = ts.createPrinter(options) + const result = nodes.map(n => printer.printNode(ts.EmitHint.Unspecified, n, source)) + + return result.join('\n') +} + +export function createSyntheticComment(text: string) { + return { + text, + pos: -1, + end: -1, + hasLeadingNewline: true, + hasTrailingNewLine: true, + kind: ts.SyntaxKind.SingleLineCommentTrivia, + } as const +} + +export function getRootDeclarationNames(text: string): string[] { + const sf = ts.createSourceFile('index.ts', text, ts.ScriptTarget.ES2020) + const names: string[] = [] + for (const s of sf.statements) { + if ((ts.isClassDeclaration(s) || ts.isFunctionDeclaration(s)) && s.name && ts.isIdentifier(s.name)) { + names.push(s.name.text) + } else if (ts.isVariableStatement(s)) { + for (const decl of s.declarationList.declarations) { + if (ts.isIdentifier(decl.name)) { + names.push(decl.name.text) + } + // Not doing binding patterns + } + } + } + return names +} + +export function getSymbolOfLastIdent(exp: ts.Expression, typeChecker: ts.TypeChecker) { + const idents = splitExpression(exp) + const termIdent = idents.pop() + if (!termIdent) { + return + } + + return typeChecker.getSymbolAtLocation(termIdent) +} + + +export function getExpressionSymbols(exp: ts.Expression, typeChecker: ts.TypeChecker) { + const idents = splitExpression(exp) + + return idents.map(n => typeChecker.getSymbolAtLocation(n)) +} + +export function createVariableStatement(name: string | ts.Identifier, exp: ts.Expression, modifiers?: ts.ModifierSyntaxKind[], factory = ts.factory) { + const modifierTokens = modifiers?.map(m => factory.createModifier(m)) + const decl = factory.createVariableDeclaration(name, undefined, undefined, exp) + + return factory.createVariableStatement( + modifierTokens, + factory.createVariableDeclarationList([decl], ts.NodeFlags.Const) + ) +} + +const importCache = new Map>() +export function getImports(source: ts.SourceFile, typeChecker: ts.TypeChecker) { + if (importCache.has(source)) { + return importCache.get(source)! + } + + // FIXME: source is `undefined` + const importDeclarations = source?.statements.filter(ts.isImportDeclaration) ?? [] + const importedSymbols = new Map() + importDeclarations.forEach(decl => { + for (const sym of getExports(decl, typeChecker)) { + importedSymbols.set(sym, decl) + } + }) + + importCache.set(source, importedSymbols) + + return importedSymbols +} + +export function getExports(decl: ts.ImportDeclaration, typeChecker: ts.TypeChecker) { + const moduleSym = typeChecker.getSymbolAtLocation(decl.moduleSpecifier) + // XXX: assert.ok(moduleSym) + // This can cause bundling to strip out things incorrectly!!!! + if (!moduleSym) { + return [] + } + + const bindings = decl.importClause?.namedBindings + if (bindings && ts.isNamespaceImport(bindings)) { + const s = typeChecker.getSymbolAtLocation(bindings.name) + + if (s) { + return [s, ...typeChecker.getExportsOfModule(moduleSym)] + } + } + + return typeChecker.getExportsOfModule(moduleSym) +} + +export function appendParameter(exp: ts.NewExpression, param: ts.Expression, factory = ts.factory) { + return factory.updateNewExpression( + exp, + exp.expression, + exp.typeArguments, + [...(exp.arguments ?? []), param] + ) +} + +export function resolveAlias(sym: ts.Symbol, typeChecker: ts.TypeChecker): ts.Symbol { + if (sym.flags & ts.SymbolFlags.Alias) { + return typeChecker.getAliasedSymbol(sym) + } + + return sym +} + +export type TypedSymbol = ts.Symbol & { readonly valueDeclaration: T } +export function isSymbolOfType( + sym: ts.Symbol | undefined, + fn: (node: ts.Node) => node is T +): sym is TypedSymbol { + return !!sym?.valueDeclaration && fn(sym.valueDeclaration) +} + +function isEnclosingNode(node: ts.Node) { + return ts.isSourceFile(node) || ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node) || ts.isClassDeclaration(node) || isMutuallyExclusiveBlockScope(node) +} + +function getScopeNode(node: ts.Node) { + const parent = ts.findAncestor(node, isEnclosingNode) + if (!parent) { + failOnNode(`No enclosing element found`, node) + } + + return parent +} + +// doesn't handle aliased identifiers yet +export function getName2(node: ts.Node): string | undefined { + if (ts.isIdentifier(node)) { + return node.text + } + + if (node.kind === ts.SyntaxKind.ThisKeyword) { + const parent = ts.findAncestor(node, ts.isClassDeclaration) ?? ts.findAncestor(node, ts.isClassExpression) ?? ts.findAncestor(node, ts.isObjectLiteralExpression) ?? ts.findAncestor(node, ts.isFunctionExpression) ?? ts.findAncestor(node, ts.isFunctionDeclaration) + if (!parent) { + failOnNode(`Failed to get container declaration`, node) + } + + return getName2(parent) + } + + if (ts.isObjectLiteralElement(node)) { + if (!node.name) { + return + } + + return getName2(node.name) + } + + if (ts.isVariableDeclaration(node) || ts.isPropertyDeclaration(node)) { + // if (!ts.isIdentifier(node.name)) { + // failOnNode(`Could not get name of node`, node) + // } + + return getName2(node.name) + } + + if (ts.isGetAccessorDeclaration(node)) { + return getName2(node.name) + } + + if (ts.isClassDeclaration(node) || ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) { + if (!node.name || !ts.isIdentifier(node.name)) { + failOnNode(`Could not get name of declaration node`, node) + } + + return getName2(node.name) + } + + if (ts.isCallExpression(node) || ts.isNewExpression(node)) { + const parts = splitExpression(node.expression) + const names = parts.map(getName2).filter(isNonNullable) + + return names.join('--') + } + + if (ts.isConstructorDeclaration(node)) { + const parent = ts.findAncestor(node, ts.isClassDeclaration) + if (!parent) { + failOnNode('No class declaration found for constructor', node) + } + + return getName2(parent)! + } + + if (ts.isPropertyAccessExpression(node)) { + return [getName2(node.expression), getName2(node.name)].join('--') + } + + if (ts.isExpressionWithTypeArguments(node)) { + const parent = ts.findAncestor(node, ts.isClassDeclaration) + if (!parent) { + failOnNode('No class declaration found for extends clause', node) + } + + return getName2(parent) + } + + if (ts.isFunctionExpression(node)) { + if (node.name) { + return getName2(node.name) + } + + return '__anonymous' + } + + if (ts.isClassExpression(node)) { + if (node.name) { + return getName2(node.name) + } + + return '__anonymous' + } + + if (node.kind === ts.SyntaxKind.SuperKeyword) { + const parent = ts.findAncestor(node, ts.isClassDeclaration) + const superClass = parent ? getSuperClassExpressions(parent)?.pop() : undefined + if (!superClass) { + failOnNode('No class declaration found when using `super` keyword', node) + } + + return getName2(superClass) + } +} + +function getNameWithDecl(node: ts.Node): string { + const baseName = getName2(node) + if (!baseName) { + failOnNode('No name', node) + } + + const parent = ts.findAncestor(node, ts.isVariableDeclaration) || ts.findAncestor(node, ts.isPropertyDeclaration) + const parentName = parent ? getName2(parent) : undefined + if (parentName !== undefined) { + return [parentName, baseName].join('--') + } + + // This is an anonymous instantiation + return baseName +} + +// Which nodes should be named? +// `CallExpression` and `NewExpression` (instantiations) +// `VariableDeclaration` and `PropertyDeclaration` (assignments) +// +// Anonymous instantiations of the same type within the same scope must be numbered +// according to their order of appearance moving from top to bottom +// +// These are only qualified w.r.t to a single instantiation scope + +function isMutuallyExclusiveBlockScope(node: ts.Node) { + if (!ts.isBlock(node)) { + return false + } + + return ts.isIfStatement(node.parent) +} + +const cachedNames = new Map() +const scopeNames = new Map>() +export function getInstantiationName(node: ts.Node) { + node = ts.getOriginalNode(node) + + if (cachedNames.has(node)) { + return cachedNames.get(node)! + } + + const name = getNameWithinScope(node) + cachedNames.set(node, name) + + return name + + function getNameWithinScope(node: ts.Node) { + // Variable statements/declarations must be unique within a scope + if (ts.isVariableStatement(node)) { + return getName2(node.declarationList.declarations[0])! + } else if (ts.isVariableDeclaration(node)) { + return getName2(node)! + } else if (ts.isClassDeclaration(node)) { + return getName2(node)! + } else if (ts.isPropertyDeclaration(node)) { + return getName2(node)! + } + + const scope = ts.getOriginalNode(getScopeNode(node)) + const name = getNameWithDecl(node) + if (!scopeNames.has(scope)) { + scopeNames.set(scope, new Set()) + } + const currentNames = scopeNames.get(scope)! + + // we only need to do this with anonymous instantiations + // if there is a variable or property decl then this is unncessary + let maybeName = name + let count = 0 + while (currentNames.has(`${maybeName}${count === 0 ? '' : `_${count}`}`)) count++ + + const finalName = `${maybeName}${count === 0 ? '' : `_${count}`}` + currentNames.add(finalName) + + return finalName + } +} + +export function getNodeLocation(node: ts.Node) { + const sf = node.getSourceFile() + if (!sf) { + return + } + + const pos = node.pos + node.getLeadingTriviaWidth(sf) + const lc = sf.getLineAndCharacterOfPosition(pos) + return `${sf.fileName}:${lc.line + 1}:${lc.character + 1}` +} + +export function failOnNode(message: string, node: ts.Node, showSourceWhenMissing = true): never { + node = ts.getOriginalNode(node) + const location = getNodeLocation(node) + if (!location && showSourceWhenMissing) { + throw new Error(`${message} [kind: ${node.kind}]\n${printNodes([node])}`) + + } + + throw new Error(`${message} [kind: ${node.kind}] (${location ?? 'missing source file'})`) +} + +export function getSuperClassExpressions(node: ts.ClassDeclaration) { + return node.heritageClauses?.filter(c => c.token === ts.SyntaxKind.ExtendsKeyword) + .map(c => [...c.types]) + .reduce((a, b) => a.concat(b), []) + .map(t => t.expression) +} + + +export function getModuleSpecifier(node: ts.ImportDeclaration | ts.ExportDeclaration): ts.StringLiteral { + if (!node.moduleSpecifier || !ts.isStringLiteral(node.moduleSpecifier)) { + failOnNode('No module specifier found', node) + } + + return node.moduleSpecifier +} + +// a little hacky but it works +export function getModuleQualifedName(node: ts.Node, typeChecker: ts.TypeChecker, rootDir: string) { + if (!ts.isExpression(node)) { + failOnNode('Not an expression', node) + } + + const sym = typeChecker.getSymbolAtLocation(node) + if (!sym) { + failOnNode(`No symbol found`, node) + } + + const imports = getImports(node.getSourceFile(), typeChecker) + const importDecl = imports.get(sym) ?? imports.get(resolveAlias(sym, typeChecker)) + if (importDecl) { + const modSpec = (importDecl.moduleSpecifier as ts.StringLiteral).text + + return `"${modSpec}".${sym.name}` + } + + const fqn = typeChecker.getFullyQualifiedName(resolveAlias(sym, typeChecker)) + const [_, location, name] = fqn.match(/"(.*)"\.(.*)/) ?? [] + if (!location || !name) { + failOnNode(`Not module-scoped: ${fqn}`, node) + } + + const moduleSegment = location.match(/.*\/node_modules\/(.*)/)?.[1] + if (!moduleSegment) { + if (location.startsWith(rootDir)) { + return `"${path.relative(rootDir, location)}".${name}` + } + + failOnNode(`Did not find module segment: ${location}`, node) + } + + + const segments = moduleSegment.split('/') + + failOnNode(`Not implemented: ${segments}`, node) +} + +export function isExported(node: ts.Node) { + return ts.canHaveModifiers(node) && !!ts.getModifiers(node)?.find(m => m.kind === ts.SyntaxKind.ExportKeyword) +} + +export function isDeclared(node: ts.Node) { + return ts.canHaveModifiers(node) && !!ts.getModifiers(node)?.find(m => m.kind === ts.SyntaxKind.DeclareKeyword) +} + +// Handles these cases: +// * Properties e.g. { : } +// * Functions e.g. { ... }> +// * Classes e.g. { ... }> +// * Variable declarations e.g. [const|let|var] = +// * Inheritance e.g. ` extends +export function inferName(node: ts.Node): string | undefined { + if (ts.isIdentifier(node) || ts.isPrivateIdentifier(node)) { + return node.text + } + + if (ts.isVariableDeclaration(node) || ts.isPropertyDeclaration(node) || ts.isPropertyAssignment(node)) { + return inferName(node.name) + } + + if (ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node)) { + if (!node.name) { + return 'default' + } + + return inferName(node.name) + } + + if (ts.isFunctionExpression(node) || ts.isClassExpression(node)) { + if (node.name) { + return inferName(node.name) + } + + return inferName(node.parent) + } + + if (ts.isArrowFunction(node)) { + return inferName(node.parent) + } + + if (node.parent && ts.isVariableDeclaration(node.parent)) { + return inferName(node.parent) + } + + if (node.parent?.parent && ts.isHeritageClause(node.parent.parent)) { + return inferName(node.parent.parent.parent) + } +} + +function getUtil(): typeof import('node:util') { + return require('node:util') +} + +export function getNullTransformationContext(): ts.TransformationContext { + if (!('nullTransformationContext' in ts)) { + const inspected = getUtil().inspect(ts, { + colors: false, + showHidden: true, + compact: true, + depth: 1, + sorted: true, + }) + + throw new Error(`No transformation context found. Current exports: ${inspected}`) + } + + return (ts as any).nullTransformationContext +} + +export async function ensureDir(dir: string) { + try { + await fs.mkdir(dir, { recursive: true }) + } catch(e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } +} + +export async function writeFile(name: string, data: string) { + await ensureDir(path.dirname(name)) + await fs.writeFile(name, data, 'utf-8') +} + +// For debugging +export function showArtifact({ name, data }: { name: string; data: any }) { + const runtime = Buffer.from(data.runtime, 'base64').toString('utf-8') + const infra = Buffer.from(data.infra, 'base64').toString('utf-8') + + return [ + `// START - ${name}`, + '', + `// ${data.source} - runtime`, + runtime, + `// ${data.source} - infrastructure`, + infra, + '', + '// END' + ].join('\n') +} + +export function hashNode(node: ts.Node) { + const original = ts.getOriginalNode(node) + const text = original.getText(original.getSourceFile()) + + return getHash(text).slice(16) +} + +export interface MemoziedFunction { + (): T + clear(): void + readonly cached: boolean +} + +export function memoize(fn: () => T): MemoziedFunction { + let didCall = false + let result: T | undefined + + function clear() { + didCall = false + result = undefined + } + + return createFunction(() => { + if (didCall) { + return result! + } + + didCall = true + return (result = fn()) + }, { + cached: { get: () => didCall }, + clear: { value: clear }, + }) +} + +type FromMap = { [P in keyof T]: T[P] extends TypedPropertyDescriptor ? U : never } + +function createFunction(fn: T, props: U): T & FromMap { + return Object.defineProperties(fn, props) as any +} + +export interface KeyedMemoziedFunction { + (...keys: K): T + delete(...keys: K): boolean + keys(): IterableIterator +} + +export function keyedMemoize(fn: (...keys: K) => T): KeyedMemoziedFunction { + const results = new Map() + const mapped = new Map() + + function getKey(keys: K) { + const key = keys.join('|') + mapped.set(key, keys) + + return key + } + + function deleteEntry(...keys: K) { + const key = getKey(keys) + mapped.delete(key) + + return results.delete(key) + } + + return createFunction( + (...keys: K) => { + const key = getKey(keys) + if (results.has(key)) { + return results.get(key)! + } + + const val = fn.apply(undefined, keys) + results.set(key, val) + + return val + }, + { + delete: { value: deleteEntry }, + keys: { value: () => mapped.values() }, + } + ) +} + +// if (val instanceof Promise) { +// const withCatch = val.catch(e => { +// results.delete(key) +// throw e +// }) as typeof val +// results.set(key, withCatch) + +// return withCatch +// } + +export async function batch(jobs: Iterable>, maxJobs = 5): Promise { + let counter = 0 + let hasNext = true + const running = new Map>() + const iter = jobs[Symbol.iterator]() + const results: T[] = [] + + while (true) { + while (hasNext && running.size < maxJobs) { + const job = iter.next() + if (job.done) { + hasNext = false + } else { + const id = counter++ + running.set(id, job.value.then(result => ({ id, result }))) + } + } + + if (running.size === 0) { + break + } + + const completed = await Promise.any(running.values()) + running.delete(completed.id) + results.push(completed.result) + } + + return results +} + +export function createSymbolPropertyName(symbol: string, factory = ts.factory) { + return factory.createCallExpression( + factory.createPropertyAccessExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier('globalThis'), + 'Symbol' + ), + 'for' + ), + + undefined, + [factory.createStringLiteral(symbol)] + ) +} + +export function removeModifiers(node: ts.Statement, modifiers: ts.ModifierSyntaxKind[], factory = ts.factory) { + if (!ts.canHaveModifiers(node)) { + return node + } + + const filteredModifiers = ts.getModifiers(node)?.filter(x => !modifiers.includes(x.kind)) + if (filteredModifiers?.length === ts.getModifiers(node)?.length) { + return node + } + + if (ts.isClassDeclaration(node)) { + return factory.updateClassDeclaration( + node, + filteredModifiers, + node.name, + node.typeParameters, + node.heritageClauses, + node.members + ) + } + + if (ts.isFunctionDeclaration(node)) { + return factory.updateFunctionDeclaration( + node, + filteredModifiers, + node.asteriskToken, + node.name, + node.typeParameters, + node.parameters, + node.type, + node.body + ) + } + + if (ts.isVariableStatement(node)) { + return factory.updateVariableStatement( + node, + filteredModifiers, + node.declarationList + ) + } + + if (ts.isEnumDeclaration(node)) { + return factory.updateEnumDeclaration( + node, + filteredModifiers, + node.name, + node.members, + ) + } + + if (ts.isTypeAliasDeclaration(node)) { + return factory.updateTypeAliasDeclaration( + node, + filteredModifiers, + node.name, + node.typeParameters, + node.type + ) + } + + if (ts.isInterfaceDeclaration(node)) { + return factory.updateInterfaceDeclaration( + node, + filteredModifiers, + node.name, + node.typeParameters, + node.heritageClauses, + node.members + ) + } + + failOnNode('Unhandled node', node) +} + +export interface AmbientDeclarationFileResult { + id: string + text: string + sourcemap: SourceMapV3 +} + +export function toAmbientDeclarationFile(moduleId: string, sourceFile: ts.SourceFile): { id: string; text: string } +export function toAmbientDeclarationFile(moduleId: string, sourceFile: ts.SourceFile, sourceMapHost: SourceMapHost, transformSpecifier?: (spec: string, importer: string) => string): AmbientDeclarationFileResult +export function toAmbientDeclarationFile(moduleId: string, sourceFile: ts.SourceFile, sourceMapHost?: SourceMapHost, transformSpecifier?: (spec: string, importer: string) => string) { + const statements = sourceFile.statements + .map(s => removeModifiers(s, [ts.SyntaxKind.DeclareKeyword])) + .map(s => { + if (!ts.isImportDeclaration(s) && !ts.isExportDeclaration(s)) { + return s + } + + const spec = (s.moduleSpecifier as ts.StringLiteral | undefined)?.text + if (!spec || !isRelativeSpecifier(spec)) { + return s + } + + const transformed = transformSpecifier?.(spec, sourceFile.fileName) + if (!transformed) { + return s + } + + if (ts.isExportDeclaration(s)) { + return ts.factory.updateExportDeclaration( + s, + s.modifiers, + false, + s.exportClause, + ts.factory.createStringLiteral(transformed, true), + s.assertClause + ) + } + + return ts.factory.updateImportDeclaration( + s, + s.modifiers, + s.importClause, + ts.factory.createStringLiteral(transformed, true), + s.assertClause + ) + }) + + const decl = ts.factory.createModuleDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.DeclareKeyword)], + ts.factory.createStringLiteral(moduleId, true), + ts.factory.createModuleBlock(statements) + ) + + const sf = ts.factory.updateSourceFile(sourceFile, [decl], true) + if (!sourceMapHost) { + const text = printNodes(sf.statements, sf, { removeComments: false }) + + return { id: moduleId, text } + } + + // TODO: map the module specifier to the first line of the source file + // Apparently the sourcemaps emitted by `tsc` don't do this so not a big deal + const { text, sourcemap } = emitChunk(sourceMapHost, sf, undefined, { emitSourceMap: true }) + + return { id: moduleId, text, sourcemap } +} + +// XXX: must be a function, otherwise `Symbol.asyncDispose` won't be initialized +function getAsyncDispose(): typeof Symbol.asyncDispose { + if (!Symbol.asyncDispose) { + const asyncDispose = Symbol.for('Symbol.asyncDispose') + Object.defineProperty(Symbol, 'asyncDispose', { value: asyncDispose, enumerable: true }) + } + + return Symbol.asyncDispose +} + +export function isErrorLike(o: unknown): o is Error { + return !!o && typeof o === 'object' && typeof (o as any).name === 'string' && typeof (o as any).message === 'string' +} + +interface LockInfo { + readonly id: string + readonly timestamp: Date +} + +const locks = new Map>() +// Very dumb lock, do not use for anything important +export async function acquireFsLock(filePath: string, maxLockDuration = 10_000) { + const id = crypto.randomUUID() + const lockFilePath = `${filePath}.lock` + + async function _readLock() { + try { + const data = JSON.parse(await fs.readFile(lockFilePath, 'utf-8')) + + return { + id: data.id, + timestamp: new Date(data.timestamp), + } + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + } + } + + function readLock() { + if (locks.has(lockFilePath)) { + return locks.get(lockFilePath)! + } + + const p = _readLock() + locks.set(lockFilePath, p) + + return p + } + + async function refreshLock() { + const d = new Date() + const data = JSON.stringify({ id, timestamp: d.toISOString() }) + await fs.writeFile(lockFilePath, data, { flag: 'w' }) + locks.set(lockFilePath, { id, timestamp: d }) + } + + async function setLock(d: Date, isExpired?: boolean) { + try { + const data = JSON.stringify({ id, timestamp: d.toISOString() }) + await fs.writeFile(lockFilePath, data, { flag: isExpired ? 'w' : 'wx' }) + const d2 = await _readLock() + if (d2?.id === id && d2.timestamp.getTime() === d.getTime()) { + return { id, timestamp: d } + } else { + locks.delete(lockFilePath) + } + } catch (e) { + if ((e as any).code !== 'EEXIST') { + throw e + } + } + } + + function checkLock(lock?: LockInfo) { + const isExpired = lock && lock.timestamp.getTime() + maxLockDuration <= Date.now() + if (isExpired === false) { + return false + } + + const d = new Date() + const p = setLock(d, isExpired) + locks.set(lockFilePath, p) + + return p.then(x => x?.id === id) + } + + function tryAcquire() { + const lockTimestamp = readLock() + if (lockTimestamp instanceof Promise) { + return lockTimestamp.then(checkLock) + } + + return checkLock(lockTimestamp) + } + + async function lock() { + if (!(await tryAcquire())) { + await new Promise((resolve, reject) => { + async function fn() { + try { + if (await tryAcquire()) { + resolve() + } else { + setTimeout(fn, 10) + } + } catch (e) { + reject(e) + } + } + + setTimeout(fn, 10) + }) + } + + return { + [getAsyncDispose()]: unlock + } + } + + async function unlock() { + await fs.rm(lockFilePath, { force: true }) + locks.delete(lockFilePath) + } + + await fs.mkdir(path.dirname(lockFilePath), { recursive: true }) + + return lock() +} + +interface TrieNode { + value?: T + readonly children: Record> +} + +function createTrieNode(value?: T): TrieNode { + return { value, children: {} } +} + +export function createTrie = string>() { + const root = createTrieNode() + + function get(key: K): T | undefined { + let node = root + for (const k of key) { + node = node.children[k] + if (!node) return + } + return node.value + } + + function insert(key: K, value: T) { + let node = root + for (const k of key) { + node = node.children[k] ??= createTrieNode() + } + node.value = value + } + + function* traverse(key: K) { + let node = root + // yield ['', node.value] as const + + for (const k of key) { + node = node.children[k] + if (!node) break + yield [k, node.value] as const + } + } + + function ancestor(key: K) { + const keys: string[] = [] + let result: T | undefined + for (const [k, v] of traverse(key)) { + keys.push(k) + result = v === undefined ? result : v + } + return result !== undefined ? [keys, result] as const : undefined + } + + function keys(key?: K) { + let node = root + if (key) { + for (const k of key) { + node = node.children[k] + if (!node) { + throw new Error(`Missing node at key part: ${k} [${key}]`) + } + } + } + + return Object.keys(node.children) + } + + function createIterator() { + let node = root + + function next(key: string) { + node = node?.children[key] + if (!node) { + return { done: true, value: undefined } + } + + return { done: false, value: node.value } + } + + return { next } + } + + return { get, insert, traverse, keys, ancestor, createIterator } +} + + +export function createHasher() { + const hashCache = new WeakMap() + + function hash(o: object) { + if (hashCache.has(o)) { + return hashCache.get(o)! + } + + const hash = getHash(JSON.stringify(o)) + hashCache.set(o, hash) + + return hash + } + + return { hash } +} + +interface BSTNode { + readonly key: K + value: V + left?: BSTNode + right?: BSTNode + parent?: BSTNode +} + +function createBSTNode(key: K, value: V): BSTNode { + return { key, value } +} + +export function createBST(compareFn?: (a: K, b: K) => number) { + type Node = BSTNode + let root: Node | undefined + const cmp = compareFn ?? ((a, b) => a < b ? -1 : a > b ? 1 : 0) + + function search(key: K): Node | undefined { + let x = root + while (x) { + const d = cmp(key, x.key) + if (d === 0) { + break + } else if (d < 0) { + x = x.left + } else { + x = x.right + } + } + + return x + } + + function insert(key: K, value: V) { + let y: Node | undefined + let x = root + while (x) { + y = x + const d = cmp(key, x.key) + if (d === 0) { + x.value = value + return + } else if (d < 0) { + x = x.left + } else { + x = x.right + } + } + + const z = createBSTNode(key, value) + z.parent = y + if (!y) { + root = z + } else if (cmp(z.key, y.key) < 0) { + y.left = z + } else { + y.right = z + } + } + + function min(n: Node): Node { + while (n.left) { + n = n.left + } + return n + } + + function successor(n: Node): Node | undefined { + if (n.right) { + return min(n.right) + } + + let y = n.parent + while (y && n === y.parent) { + n = y + y = y.parent + } + return y + } + + function remove(key: K): V | undefined { + const n = search(key) + if (!n) { + return + } + + if (!n.left) { + shiftNodes(n, n.right) + } else if (!n.right) { + shiftNodes(n, n.left) + } else { + const s = successor(n)! + if (s.parent !== n) { + shiftNodes(s, s.right) + s.right = n.right + s.right.parent = s + } + shiftNodes(n, s) + s.left = n.left + s.left.parent = s + } + + return n.value + } + + function shiftNodes(x: Node, y: Node | undefined) { + if (!x.parent) { + root = y + } else if (x === x.parent.left) { + x.parent.left = y + } else { + x.parent.right = y + } + if (y) { + y.parent = x.parent + } + } + + function find(key: K): V | undefined { + return search(key)?.value + } + + return { + find, + insert, + remove, + } +} + +export function createMinHeap(compareFn?: (a: T, b: T) => number, initArray: T[] = []) { + const a = initArray + const cmp = compareFn ?? ((a, b) => a < b ? -1 : a > b ? 1 : 0) + + function swap(i: number, j: number) { + const tmp = a[i] + a[i] = a[j] + a[j] = tmp + } + + function insert(element: T) { + a.push(element) + siftUp(a.length - 1) + } + + function extract(): T { + if (a.length === 0) { + throw new Error('Empty') + } + + const v = a[0] + const u = a.pop()! + + if (a.length > 0) { + a[0] = u + minHeapify(0) + } + + return v + } + + function siftUp(i: number): void { + if (i === 0) { + return + } + + const j = Math.floor((i - 1) / 2) + if (cmp(a[i], a[j]) < 0) { + swap(i, j) + siftUp(j) + } + } + + function minHeapify(i: number) { + const left = 2 * i + 1 + const right = left + 1 + let smallest = i + if (left < a.length && cmp(a[left], a[smallest]) < 0) { + smallest = left + } + if (right < a.length && cmp(a[right], a[smallest]) < 0) { + smallest = right + } + if (smallest !== i) { + swap(i, smallest) + minHeapify(smallest) + } + } + + return { + insert, + extract, + get length() { + return a.length + } + } +} + +interface LockData { + readonly type: 'read' | 'write' + readonly callback?: () => void + acquired?: boolean +} + +export function createRwMutex() { + let idCounter = 0 + const data = new Map() + + function getAcquired() { + return [...data.values()].filter(v => v.acquired) + } + + function release(id: number) { + if (!data.has(id)) { + return + } + + const lastLockType = data.get(id)!.type + data.delete(id) + + const readers = getAcquired().filter(x => x.type === 'read') + let nextLockType = readers.length === 0 ? 'write' : lastLockType + + for (const l of data.values()) { + if (l.acquired) continue + + if (nextLockType === 'write' || l.type === 'read') { + l.acquired = true + l.callback?.() + if (l.type === 'write') { + break + } + nextLockType = 'read' + } + } + } + + function acquire(type: 'read' | 'write') { + const id = idCounter++ + const acquired = getAcquired() + const incompatibleLocks = type === 'read' + ? acquired.filter(x => x.type === 'write') + : acquired + + const l = { dispose: () => release(id) } + + if (incompatibleLocks.length > 0) { + return new Promise<{ dispose: () => void }>((resolve, reject) => { + const lock: LockData = { + type, + callback: () => resolve(l), + } + data.set(id, lock) + }) + } + + const lock: LockData = { + type, + acquired: true, + } + + data.set(id, lock) + + return Promise.resolve(l) + } + + function lockRead() { + return acquire('read') + } + + function lockWrite() { + return acquire('write') + } + + return { lockRead, lockWrite } +} + +interface FileHasherCache { + [file: string]: { hash: string; mtime: number } +} + +// This is should ideally only be used for source files +export function createFileHasher(fs: Pick, cacheLocation: string) { + async function loadCache(): Promise { + try { + return JSON.parse(await fs.readFile(path.resolve(cacheLocation, 'files.json'), 'utf-8')) + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e + } + return {} + } + } + + async function saveCache(data: FileHasherCache) { + const existing = await loadCache() + for (const [k, v] of Object.entries(existing)) { + if (!data[k] || data[k].mtime < v.mtime) { + data[k] = v + } + } + await fs.writeFile(path.resolve(cacheLocation, 'files.json'), JSON.stringify(data)) + } + + const getCache = memoize(loadCache) + + async function _getHash(fileName: string) { + const { hash } = await checkFile(fileName) + + return hash + } + + async function checkFile(fileName: string) { + const cachePromise = getCache() + const stat = await fs.stat(fileName).catch(async e => { + delete (await cachePromise)[fileName] + throw e + }) + + const cache = await cachePromise + const cached = cache[fileName] + if (cached && cached.mtime === stat.mtimeMs) { + return { hash: cached.hash } + } + + const data = await fs.readFile(fileName) + const hash = getHash(data) + cache[fileName] = { mtime: stat.mtimeMs, hash } + + return { hash } + } + + async function flush() { + await saveCache(await getCache()) + getCache.clear() + } + + return { getHash: _getHash, checkFile, flush } +} + +export async function makeExecutable(fileName: string) { + await fs.chmod(fileName, 0o755) +} + +export async function linkBin(existingPath: string, newPath: string) { + await ensureDir(path.dirname(newPath)) + await fs.unlink(newPath).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + }) + await fs.symlink(existingPath, newPath, 'file') + await makeExecutable(newPath) +} + +function filterObject(obj: Record, fn: (v: any) => boolean) { + return Object.fromEntries(Object.entries(obj).filter(([_, v]) => fn(v))) +} + +export function splitObject< + T extends Record, + U extends T[keyof T] +>(obj: T, fn: (val: T[keyof T]) => val is U): { left: Record; right: Record> } { + return { + left: filterObject(obj, fn), + right: filterObject(obj, v => !fn(v)), + } +} + +export function escapeRegExp(pattern: string) { + return pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} + +// SLOW!!! +export function dedupe(statements: any[]) { + return Object.values(Object.fromEntries(statements.map(s => [JSON.stringify(s), s]))) +} + +export function getCiType(): 'github' | undefined { + const env = process.env + + if (env['CI'] && env['GITHUB_REPOSITORY']) { + return 'github' + } +} + +export function isWindows() { + return process.platform === 'win32' +} + +// TERM_PROGRAM=vscode +// TERM_PROGRAM_VERSION=1.87.2 + +const knownVscEnvVars = [ + 'VSCODE_GIT_ASKPASS_NODE', + 'VSCODE_GIT_ASKPASS_MAIN', + 'VSCODE_GIT_ASKPASS_EXTRA_ARGS', + 'VSCODE_GIT_IPC_HANDLE', + 'VSCODE_INJECTION', +] + +export function isRunningInVsCode() { + return process.env['TERM_PROGRAM'] === 'vscode' +} + +export function replaceWithTilde(filePath: string) { + const dir = homedir() + if (filePath.startsWith(dir)) { + return filePath.replace(dir, '~') + } + + return filePath +} + +export function gzip(data: string | ArrayBuffer) { + return new Promise((resolve, reject) => { + zlib.gzip(data, (err, res) => err ? reject(err) : resolve(res)) + }) +} + +export function gunzip(data: string | ArrayBuffer) { + return new Promise((resolve, reject) => { + zlib.gunzip(data, (err, res) => err ? reject(err) : resolve(res)) + }) +} + +export function isRelativeSpecifier(spec: string) { + return spec === '.' || spec.startsWith('./') || spec.startsWith('../') || spec === '..' +} + +// `localeCompare` calls `new Intl.Collator` which can take a bit of time to create + +export function strcmp(a: string, b: string) { + return a < b ? -1 : a > b ? 1 : 0 +} + +export function sortRecord(record: Record): Record { + return Object.fromEntries(Object.entries(record).sort((a, b) => strcmp(a[0], b[0]))) +} + +export function filterRecord(record: Record, fn: (k: string, v: T) => boolean): Record { + return Object.fromEntries(Object.entries(record).filter(([k, v]) => fn(k, v))) +} + +export function throwIfNotFileNotFoundError(err: unknown): asserts err is Error & { code: 'ENOENT' } { + if (util.types.isNativeError(err) && (err as any).code !== 'ENOENT') { + throw err + } +} + +export async function tryReadJson(fs: Pick, fileName: string) { + try { + return JSON.parse(await fs.readFile(fileName, 'utf8')) as T + } catch (e) { + throwIfNotFileNotFoundError(e) + } +} + +export function tryReadJsonSync(fs: Pick, fileName: string) { + try { + return JSON.parse(fs.readFileSync(fileName, 'utf8')) as T + } catch (e) { + throwIfNotFileNotFoundError(e) + } +} + +export type Mutable = { -readonly [P in keyof T]: T[P] } + +// Terraform stores attributes with `_` instead of json so we need to normalize them +export const capitalize = (s: string) => s ? s.charAt(0).toUpperCase().concat(s.slice(1)) : s +export const uncapitalize = (s: string) => s ? s.charAt(0).toLowerCase().concat(s.slice(1)) : s + +export function toSnakeCase(str: string) { + const pattern = /[A-Z]/g + const parts: string[] = [] + + let lastIndex = 0 + let match: RegExpExecArray | null + while (match = pattern.exec(str)) { + parts.push(str.slice(lastIndex, match.index)) + lastIndex = match.index + } + + if (lastIndex !== str.length) { + parts.push(str.slice(lastIndex, str.length)) + } + + return parts.map(uncapitalize).join('_') +} + +const defaultHashEncoding: 'base64' | 'base64url' | 'hex' | 'binary' | 'none' = 'hex' // Case-insensitive file systems make hex a better choice +export function getHash(data: string | Buffer | Uint8Array, enc?: 'base64' | 'base64url' | 'hex' | 'binary', alg?: 'sha256' | 'sha512'): string +export function getHash(data: string | Buffer | Uint8Array, enc: 'none', alg?: 'sha256' | 'sha512'): Buffer +export function getHash(data: string | Buffer | Uint8Array, enc = defaultHashEncoding, alg: 'sha256' | 'sha512' = 'sha256') { + if ('hash' in crypto) { + return (crypto as any).hash(alg, data, enc === 'none' ? 'buffer' : enc) + } + + const hash = (crypto as any).createHash(alg).update(data) + return enc !== 'none' ? hash.digest(enc) : hash.digest() +} + +export function getArrayHash(data: (string | Buffer | Uint8Array)[], enc: 'hex' = 'hex', alg: 'sha256' | 'sha512' = 'sha256') { + const hash = crypto.createHash(alg) + hash.update(`__array${data.length}__`) + for (let i = 0; i < data.length; i++) { + hash.update(data[i]) + } + return hash.digest(enc) +} + +export async function tryFindFile(fs: Pick, fileName: string, startDir: string, endDir?: string) { + try { + return { + data: await fs.readFile(path.resolve(startDir, fileName)), + directory: startDir, + } + } catch (e) { + if ((e as any).code !== 'ENOENT') { + throw e // TODO: add lookup stack + } + + const nextDir = path.dirname(startDir) + if (nextDir === endDir || nextDir === startDir) { + return + } + + return tryFindFile(fs, fileName, nextDir, endDir) + } +} + +type Proxied = { [P in keyof T]+?: T[P] } + +// Very simple wrapper. Can easily be bypassed through property descriptors +export function wrapWithProxy(target: T, proxied: Proxied): T { + const s = new Set([...Object.keys(proxied), ...Object.getOwnPropertySymbols(proxied)]) + + function get(t: any, prop: PropertyKey, recv: any): any { + if (s.has(prop)) { + return proxied[prop as keyof T] + } + + return Reflect.get(t, prop, recv) + } + + return new Proxy(target, { get }) +} + +// EDIT-DISTANCE UTILS + +// Wagner–Fischer algorithm +export function levenshteinDistance(a: string, b: string) { + if (a === b) { + return 0 + } else if (!a) { + return b.length + } else if (!b) { + return a.length + } + + const m = a.length + 1 + const n = b.length + 1 + + const dists: number[][] = [] + for (let i = 0; i < m; i++) { + dists[i] = [] + for (let j = 0; j < n; j++) { + dists[i][j] = 0 + } + } + + for (let i = 1; i < m; i++) { + dists[i][0] = i + } + + for (let j = 1; j < n; j++) { + dists[0][j] = j + } + + for (let i = 1; i < m; i++) { + for (let j = 1; j < n; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1 + dists[i][j] = Math.min( + dists[i - 1][j] + 1, + dists[i][j - 1] + 1, + dists[i - 1][j - 1] + cost, + ) + } + } + + return dists[a.length][b.length] +} + +interface CostFunctions { + insert: (left: T) => number + update: (left: T, right: T) => number + remove: (right: T) => number +} + +type InsertOp = ['insert', T] +type UpdateOp = ['update', T, T] +type RemoveOp = ['remove', T] +type NoOp = ['noop', T, T] + +type EditOp = + | InsertOp + | UpdateOp + | RemoveOp + | NoOp + +export function arrayEditDistance(a: T[], b: T[], costs: Partial>) { + // if (a === b) { + // return 0 + // } else if (!a) { + // return b.length + // } else if (!b) { + // return a.length + // } + + const m = a.length + 1 + const n = b.length + 1 + const dists: number[][] = [] + const ops: EditOp[][][] = [] + const { insert, update, remove } = costs + + for (let i = 0; i < m; i++) { + dists[i] = [] + ops[i] = [] + for (let j = 0; j < n; j++) { + dists[i][j] = 0 + ops[i][j] = [] + } + } + + for (let i = 1; i < m; i++) { + const left = a[i - 1] + dists[i][0] = !insert ? i : dists[i - 1][0] + insert(left) + ops[i][0] = [...ops[i - 1][0], ['insert', left]] + } + + for (let j = 1; j < n; j++) { + const right = b[j - 1] + dists[0][j] = !remove ? j : dists[0][j - 1] + remove(right) + ops[0][j] = [...ops[0][j - 1], ['remove', right]] + } + + for (let i = 1; i < m; i++) { + for (let j = 1; j < n; j++) { + const left = a[i - 1] + const right = b[j - 1] + const c = [ + dists[i - 1][j] + (insert ? insert(left) : 1), + dists[i][j - 1] + (remove ? remove(right) : 1), + dists[i - 1][j - 1] + (left === right ? 0 : update ? update(left, right) : 1), + ] + + const m = dists[i][j] = Math.min(c[0], c[1], c[2]) + + switch (c.indexOf(m)) { + case 0: + ops[i][j] = [...ops[i - 1][j], ['insert', left]] + break + case 1: + ops[i][j] = [...ops[i][j - 1], ['remove', right]] + break + case 2: + ops[i][j] = [...ops[i - 1][j - 1], [left === right ? 'noop' : 'update', left, right]] + break + } + } + } + + return { + ops: ops[a.length][b.length], + score: dists[a.length][b.length], + } +} + + +// We only need to determine a unique name for a resource when the program is ran to +// generate the infrastructure configuration. Before that, names can be represented +// symbolically. +/// +// Goals +// * Names are unique within their scope +// * Creating names is deterministic +// * Changing the basic program structure should not change the names +// * That is, changing the program surrounding a resource declaration should not change its name +// +// Resource name algorithm +// 1. Determine the scope (i.e. the parent node) +// a) PropertyDeclaration, VariableDeclaration +// * If destructuring, treat this as an 'anonymous' variable +// b) ClassDeclaration, FunctionDeclaration, FunctionExpression, ClassExpression +// c) ArrowFunction (similar to FunctionExpression) +// d) ForStatement +// e) IfStatement (`else` block is distinct from the primary block) +// f) SourceFile (root scope for a JS module but not application) +// 2. Within the scope, give the resource a unique name +// a) Handle collisions by appending a count based off order of appeareance e.g. `R_1`, `R_2`, etc. +// + + +// Examples +// +// ```ts +// class X {} +// const y = new X() +// ``` +// +// After refactoring to +// ```ts +// class X {} +// const y = foo() +// function foo() { +// return new X() +// } +// ``` +// Ideally, we should interpret this as the _same_ program because it _is_ the same +// once executed. We need to perform a "scope relaxation" phase after qualifying all +// names +// +// Following the above example: +// ```ts +// class R1 {} +// class R2 {} +// const x = foo() +// const y = foo() +// const z = { a: foo(), b: foo() } +// function foo() { +// const x = new R2() +// return new R1() +// } +// ``` +// +// When looking at only the `foo` block we would see this: +// * foo/x/R2 (R2) +// * foo/R1 (R1) +// +// At the top level we can describe the resources like so: +// * x/R1 (R1) +// * x/R2 (R2) +// * y/R1 (R1) +// * y/R2 (R2) +// * z/a/R1 (R1) +// * z/a/R2 (R2) +// * z/b/R1 (R1) +// * z/b/R2 (R2) +// +// Open question: should only variable/property names be used when deriving the path? +// Open question: will path contraction cause unexpected retention of resources? +// * Probably not. If path contraction results in the same names then at worse the resource will updated + + +// Only clones: +// * enumerable string props +// * array, maps, and sets +export function deepClone(val: T, visited = new Map()): T { + // Doesn't handle bound functions + if (typeof val === 'function') { + return val + } + + if (typeof val !== 'object' || !val) { + return val + } + + if (visited.has(val)) { + return visited.get(val) + } + + if (Array.isArray(val)) { + const arr: any = [] + visited.set(val, arr) + for (let i = 0; i < val.length; i++) { + arr[i] = deepClone(val[i], visited) + } + + return arr as T + } + + if (val instanceof Map) { + return new Map(val.entries()) as T + } + + if (val instanceof Set) { + return new Set(val.values()) as T + } + + const res = {} as any + visited.set(val, res) + for (const [k, v] of Object.entries(val)) { + res[k] = deepClone(v, visited) + } + + return res as T +} + +export function makeRelative(from: string, to: string) { + const result = path.relative(from, to) + if (process.platform !== 'win32') { + return result + } + return result.replaceAll('\\', '/') +} + +export function resolveRelative(from: string, to: string) { + const result = path.resolve(from, to) + if (process.platform !== 'win32') { + return result + } + return result.replaceAll('\\', '/') +} \ No newline at end of file diff --git a/src/utils/caches.ts b/src/utils/caches.ts new file mode 100644 index 0000000..ab5e199 --- /dev/null +++ b/src/utils/caches.ts @@ -0,0 +1,63 @@ +import { memoize } from '../utils' +import { Memento } from './memento' + +export interface TtlCache extends Omit { + /** `ttl` is in seconds */ + set(key: string, value: T, ttl: number): Promise +} + +export function createTtlCache(memento: Memento): TtlCache { + const manifestKey = '__ttl-manifest__' + + function _getManifest() { + return memento.get>(manifestKey, {}) + } + + // We're assuming there is only 1 writer per-memento + const getManifest = memoize(_getManifest) + + async function updateManifest(entries: Record) { + const m = await getManifest() + await memento.set(manifestKey, { ...m, ...entries }) + } + + async function putEntry(key: string, ttl: number) { + await updateManifest({ [key]: Date.now() + (ttl * 1000) }) + } + + async function deleteEntry(key: string) { + await updateManifest({ [key]: undefined }) + } + + async function isInvalid(key: string) { + const m = await getManifest() + if (!m[key]) { + return // Maybe always return true here? + } + + return Date.now() >= m[key] + } + + async function get(key: string, defaultValue?: T): Promise { + if (await isInvalid(key) === true) { + await deleteEntry(key) + await memento.delete(key) + + return defaultValue + } + + return memento.get(key, defaultValue) + } + + async function set(key: string, value: T, ttl: number) { + await putEntry(key, ttl) + await memento.set(key, value) + } + + async function _delete(key: string) { + await deleteEntry(key) + await memento.delete(key) + } + + return { get, set, delete: _delete } +} \ No newline at end of file diff --git a/src/utils/convertNodePrimordials.ts b/src/utils/convertNodePrimordials.ts new file mode 100644 index 0000000..320358d --- /dev/null +++ b/src/utils/convertNodePrimordials.ts @@ -0,0 +1,1002 @@ +import ts from 'typescript' +import * as path from 'node:path' +import { glob } from './glob' +import { getFs, isCancelled, throwIfCancelled } from '../execution' +import { Symbol, createGraphCompiler, getRootSymbol, printNodes } from '../static-solver' +import { Mutable, failOnNode, getNodeLocation, getNullTransformationContext } from '../utils' +import { runCommand } from './process' +import { getLogger } from '..' + +interface Mappings { + instanceFields: Record + staticFields: Record + instanceGet: Record + instanceSet: Record + transforms?: Record +} + +interface Matcher { + readonly symbolName?: string + readonly matchType: ts.SyntaxKind | (ts.SyntaxKind | string)[] + readonly fn: (n: ts.Node) => ts.Node +} + +function createAliasTransform(name: string): Matcher { + return { + matchType: ts.SyntaxKind.Identifier, + fn: () => ts.factory.createIdentifier(name), + } +} + +function createPromiseMapper(name: string): Matcher { + return { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + if (!ts.isCallExpression(n)) { + failOnNode('Not a call expression', n) + } + + const t = ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Promise'), + name + ) + + const arg = n.arguments.length === 2 + ? ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + n.arguments[0], + 'map' + ), + undefined, + [n.arguments[1]] + ) + : n.arguments[0] + + return ts.factory.createCallExpression( + t, + undefined, + [arg] + ) + }, + } +} + +function createIterTransform(): Matcher { + return { + matchType: [ts.SyntaxKind.NewExpression], + fn: (node: ts.Node) => { + if (ts.isNewExpression(node)) { + return node.arguments![0] + } + + if (ts.isSpreadElement(node) && ts.isNewExpression(node.expression)) { + return ts.factory.updateSpreadElement(node, node.expression.arguments![0]) + } + + if (!ts.isForOfStatement(node)) { + failOnNode('wrong node', node) + } + + if (!ts.isNewExpression(node.expression) || !node.expression.arguments) { + failOnNode('wrong node', node) + } + + return ts.factory.updateForOfStatement( + node, + node.awaitModifier, + node.initializer, + node.expression.arguments[0], + node.statement, + ) + }, + } +} + +function Identity(): Matcher { + return { + matchType: ts.SyntaxKind.Identifier, + fn: n => n, + } +} + +function SetAccessorReplacer(prop: string, symbolName?: string): Matcher { + return { + symbolName, + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + if (!ts.isCallExpression(n)) { + failOnNode('Not a call expression', n) + } + + return ts.factory.createAssignment( + ts.factory.createPropertyAccessExpression( + n.arguments[0], + prop, + ), + n.arguments[1], + ) + } + } +} + +function GetAccessorReplacer(prop: string, symbolName?: string): Matcher { + return { + symbolName, + matchType: [ts.SyntaxKind.CallExpression, 'expression'], + fn: (n: ts.Node) => { + if (!ts.isCallExpression(n)) { + failOnNode('Not a call expression', n) + } + + return ts.factory.createPropertyAccessExpression( + n.arguments[0], + prop, + ) + } + } +} + +function KnownSymbolReplacer(name: string) { + return { + matchType: ts.SyntaxKind.Identifier, + fn: (n: ts.Node) => { + return ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Symbol'), + `Symbol.${name}` + ) + } + } +} + +function SymbolReplacer(name: string) { + return { + matchType: ts.SyntaxKind.Identifier, + fn: (n: ts.Node) => { + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Symbol'), + 'for', + ), + undefined, + [ts.factory.createStringLiteral(name)] + ) + } + } +} + +// These fields will have an additional `Apply` mapping +const varargsMethods = new Set([ + 'ArrayOf', + 'ArrayPrototypePush', + 'ArrayPrototypeUnshift', + 'MathHypot', + 'MathMax', + 'MathMin', + 'StringFromCharCode', + 'StringFromCodePoint', + 'StringPrototypeConcat', + 'TypedArrayOf', +]) + +const intrinsics = [ + 'AggregateError', + 'Array', + 'ArrayBuffer', + 'BigInt', + 'BigInt64Array', + 'BigUint64Array', + 'Boolean', + 'DataView', + 'Date', + 'Error', + 'EvalError', + 'FinalizationRegistry', + 'Float32Array', + 'Float64Array', + 'Function', + 'Int16Array', + 'Int32Array', + 'Int8Array', + 'Map', + 'Number', + 'Object', + 'RangeError', + 'ReferenceError', + 'RegExp', + 'Set', + 'String', + 'Symbol', + 'SyntaxError', + 'TypeError', + 'URIError', + 'Uint16Array', + 'Uint32Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'WeakMap', + 'WeakRef', + 'WeakSet', + + 'Promise', +] + +// TODO: primordials[fallback] -> globalThis[fallback] +// IMPORTANT: `lib/buffer.js` suffers a perf regression without pre-binding `fill` +// const Fill = Uint8Array.prototype.fill.call.bind(Uint8Array.prototype.fill) +// TypedArrayGetToStringTag(value) +// TODO: customize build system to only build the executable +// Object.prototype.toString.call(val2); +// ReferenceError: StringPrototypePadStart is not defined -> String.prototype.padStart.call.bind(String.prototype.padStart) +// case kWebCryptoCipherDecrypt: { +// const slice = ArrayBuffer.isView(data) ? +// TypedArrayPrototypeSlice : ArrayBufferPrototypeSlice; + +// ErrorPrototypeToString + +// `lib/internal/error_serdes.js:83` +// const ObjectPrototypeToString = Object.prototype.toString.call.bind(Object.prototype.toString) + + +// This is currently missed +// (ctx.showHidden ? +// ObjectPrototypeHasOwnProperty : +// ObjectPrototypePropertyIsEnumerable) + +// function arrayBufferViewTypeToIndex(abView) { +// const type = abView.toString(); +// Needs to use `const ObjectPrototypeToString = Object.prototype.toString.call.bind(Object.prototype.toString);` + +// lib/internal/assert/assertion_error.js:381 +// Object.defineProperty(this, 'name', { +// __proto__: null, // <-- this is needed ??? +// value: 'AssertionError [ERR_ASSERTION]', +// enumerable: false, +// writable: true, +// configurable: true +// }); + +// node:internal/util/comparisons:240 +// if (!val2.propertyIsEnumerable(key)) { +// ^ + +// TypeError: val2.propertyIsEnumerable is not a function +// const ObjectPrototypePropertyIsEnumerable = Object.prototype.propertyIsEnumerable.call.bind(Object.prototype.propertyIsEnumerable); + +// if (stringified.startsWith('class') && stringified.endsWith('}')) { +// const neverIndex = (map[kSensitiveHeaders] || emptyArray).map(StringPrototypeToLowerCase); +// actual: TypeError: oneLineNamedImports.replace is not a function + +// val1.valueOf is not a function + + +// The behavior of `net.Socket` close event is different + +// const stringified = value.toString(); +// if (stringified.startsWith('class') && stringified.endsWith('}')) { +// FunctionPrototypeToString = Function.prototype.toString.call.bind(Function.prototype.toString) + +// const NumberPrototypeValueOf = Number.prototype.valueOf.call.bind(Number.prototype.valueOf) +// const StringPrototypeValueOf = String.prototype.valueOf.call.bind(String.prototype.valueOf) +// const BooleanPrototypeValueOf = Boolean.prototype.valueOf.call.bind(Boolean.prototype.valueOf) +// const BigIntPrototypeValueOf = BigInt.prototype.valueOf.call.bind(BigInt.prototype.valueOf) +// const SymbolPrototypeValueOf = Symbol.prototype.valueOf.call.bind(Symbol.prototype.valueOf) + +// function isEqualBoxedPrimitive(val1, val2) { +// if (isNumberObject(val1)) { +// return isNumberObject(val2) && +// Object.is(NumberPrototypeValueOf(val1), NumberPrototypeValueOf(val2)); +// } +// if (isStringObject(val1)) { +// return isStringObject(val2) && +// StringPrototypeValueOf(val1) === StringPrototypeValueOf(val2); +// } +// if (isBooleanObject(val1)) { +// return isBooleanObject(val2) && +// BooleanPrototypeValueOf(val1) === BooleanPrototypeValueOf(val2); +// } +// if (isBigIntObject(val1)) { +// return isBigIntObject(val2) && +// BigIntPrototypeValueOf(val1) === BigIntPrototypeValueOf(val2); +// } +// if (isSymbolObject(val1)) { +// return isSymbolObject(val2) && +// SymbolPrototypeValueOf(val1) === SymbolPrototypeValueOf(val2); +// } + +// /opt/homebrew/opt/node@21/bin/node ./benchmark/compare2.js --old /opt/homebrew/bin/node --filter buffer-base64 buffers +// ./benchmark/compare2.js --old /opt/homebrew/opt/node/bin/node --filter multi-buffer dgram + +function RegExpStringMethodReplacer(methodName: string): Matcher { + return { + matchType: ts.SyntaxKind.CallExpression, + fn: (n) => { + assertCallExpression(n) + + const args = methodName === 'replace' + ? [n.arguments[0], n.arguments[2] ?? ts.factory.createStringLiteral('')] // Not providing a 2nd arg is a bug, `inspect.js` has this bug + : [n.arguments[0], ...n.arguments.slice(2)] + + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression(n.arguments[1], methodName), + undefined, + args + ) + }, + } +} + +function assertCallExpression(n: ts.Node): asserts n is ts.CallExpression { + if (!ts.isCallExpression(n)) { + failOnNode('Not a call expression', n) + } +} + +function propertyAccess(target: ts.Expression | string, member: string) { + const t = typeof target === 'string' ? ts.factory.createIdentifier(target) : target + + return ts.factory.createPropertyAccessExpression(t, member) +} + +function createExtras() { + + const extras: Partial = { + staticFields: { + PromiseResolve: 'Promise.resolve', + PromiseReject: 'Promise.reject', + + }, + instanceFields: { + SafeStringPrototypeSearch: 'search', + SafePromisePrototypeFinally: 'finally', + }, + transforms: { + hardenRegExp: { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + assertCallExpression(n) + + if (n.arguments.length === 0) { + failOnNode('Expected at least 1 argument', n) + } + return n.arguments[0] + }, + }, + + ObjectPrototypeHasOwnProperty: { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + assertCallExpression(n) + + return ts.factory.updateCallExpression( + n, + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Object'), + 'hasOwn' + ), + undefined, + n.arguments + ) + }, + }, + + SafePromiseAny: createPromiseMapper('any'), + SafePromiseAll: createPromiseMapper('all'), + SafePromiseRace: createPromiseMapper('race'), + SafePromiseAllReturnVoid: createPromiseMapper('all'), + SafePromiseAllReturnArrayLike: createPromiseMapper('all'), + SafePromiseAllSettledReturnVoid: createPromiseMapper('allSettled'), + + SafeSet: createAliasTransform('Set'), + SafeMap: createAliasTransform('Map'), + SafeWeakRef: createAliasTransform('WeakRef'), + SafeWeakSet: createAliasTransform('WeakSet'), + SafeWeakMap: createAliasTransform('WeakMap'), + SafeFinalizationRegistry: createAliasTransform('FinalizationRegistry'), + MainContextError: createAliasTransform('Error'), + + SafeStringIterator: createIterTransform(), + SafeArrayIterator: createIterTransform(), + + RegExpPrototypeSymbolReplace: RegExpStringMethodReplacer('replace'), + RegExpPrototypeSymbolSplit: RegExpStringMethodReplacer('split'), + RegExpPrototypeSymbolSearch: RegExpStringMethodReplacer('search'), + + SymbolDispose: SymbolReplacer('dispose'), + SymbolAsyncDispose: SymbolReplacer('Symbol.asyncDispose'), + SymbolIterator: KnownSymbolReplacer('iterator'), + SymbolAsyncIterator: KnownSymbolReplacer('asyncIterator'), + SymbolHasInstance: KnownSymbolReplacer('hasInstance'), + + // This overrides the default because we rarely want to call the instance method directly + FunctionPrototypeSymbolHasInstance: { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + assertCallExpression(n) + + return ts.factory.createCallExpression( + propertyAccess( + ts.factory.createElementAccessExpression( + propertyAccess('Function', 'prototype'), + propertyAccess('Symbol', 'hasInstance') + ), + 'call' + ), + undefined, + n.arguments + ) + }, + }, + + // `buffer.js` uses this as a "super" call so we can't call the instance method + TypedArrayPrototypeFill: { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + assertCallExpression(n) + + return ts.factory.createCallExpression( + propertyAccess(propertyAccess(propertyAccess('Uint8Array', 'prototype'), 'fill'), 'call'), + undefined, + n.arguments + ) + } + }, + + // `arguments.slice` isn't valid + ArrayPrototypeSlice: { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + assertCallExpression(n) + + if (n.arguments.length > 0 && ts.isIdentifier(n.arguments[0]) && n.arguments[0].text === 'arguments') { + return ts.factory.createCallExpression( + propertyAccess(propertyAccess(propertyAccess('Array', 'prototype'), 'slice'), 'call'), + undefined, + n.arguments + ) + } + + return transformMethod(n, 'slice') + } + }, + + TypedArrayPrototypeGetSymbolToStringTag: { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + assertCallExpression(n) + + return ts.factory.createElementAccessExpression( + n.arguments[0], + propertyAccess('Symbol', 'toStringTag') + ) + } + }, + + // func => Function.prototype.call.bind(func) + uncurryThis: { + matchType: ts.SyntaxKind.CallExpression, + fn: (n: ts.Node) => { + assertCallExpression(n) + + return ts.factory.createCallExpression( + propertyAccess( + propertyAccess( + propertyAccess('Function', 'prototype'), + 'call' + ), + 'bind' + ), + undefined, + n.arguments, + ) + } + }, + + IteratorPrototype: { + matchType: ts.SyntaxKind.Identifier, + fn: () => { + const factory = ts.factory + return factory.createCallExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier("Reflect"), + factory.createIdentifier("getPrototypeOf") + ), + undefined, + [factory.createCallExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier("Reflect"), + factory.createIdentifier("getPrototypeOf") + ), + undefined, + [factory.createCallExpression( + factory.createElementAccessExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier("Array"), + factory.createIdentifier("prototype") + ), + factory.createPropertyAccessExpression( + factory.createIdentifier("Symbol"), + factory.createIdentifier("iterator") + ) + ), + undefined, + [] + )] + )] + ) + }, + }, + + AsyncIteratorPrototype: { + matchType: ts.SyntaxKind.Identifier, + fn: () => { + const factory = ts.factory + return factory.createCallExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier("Reflect"), + factory.createIdentifier("getPrototypeOf") + ), + undefined, + [factory.createPropertyAccessExpression( + factory.createCallExpression( + factory.createPropertyAccessExpression( + factory.createIdentifier("Reflect"), + factory.createIdentifier("getPrototypeOf") + ), + undefined, + [factory.createFunctionExpression( + [factory.createToken(ts.SyntaxKind.AsyncKeyword)], + factory.createToken(ts.SyntaxKind.AsteriskToken), + undefined, + undefined, + [], + undefined, + factory.createBlock( + [], + false + ) + )] + ), + factory.createIdentifier("prototype") + )] + ) + }, + } + + // deprecated + // escape: Identity(), + // eval: Identity(), + // unescape: Identity(), + + // IteratorPrototype: Reflect.getPrototypeOf(primordials.ArrayIteratorPrototype); + } + } + + return extras +} + +function isMatch(n: ts.Node, matcher: Matcher, getSymbol: (n: ts.Node) => Symbol | undefined): boolean { + if (!Array.isArray(matcher.matchType)) { + return n.kind === matcher.matchType + } + + let c: any = n + for (const p of matcher.matchType) { + if (typeof p === 'string') { + if (!(p in c)) return false + c = c[p] + if (!c) return false + } else { + if (c.kind !== p) return false + } + } + + if (matcher.symbolName) { + const sym = getSymbol(c) + if (sym?.name !== matcher.symbolName) { + return false + } + + const init = sym.variableDeclaration?.initializer + if (!init || !(ts.isIdentifier(init) && init.text === 'primordials')) { + return false + } + + } + + return true +} + +function transformMethod(n: ts.Node, v: string) { + assertCallExpression(n) + + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + n.arguments[0], + v, + ), + undefined, + [...n.arguments.slice(1)] + ) +} + +function isProtoNull(prop: ts.ObjectLiteralElementLike) { + if (!ts.isPropertyAssignment(prop)) { + return false + } + + return ( + (ts.isIdentifier(prop.name) || ts.isStringLiteral(prop.name)) && + prop.name.text === '__proto__' && + prop.initializer.kind === ts.SyntaxKind.NullKeyword + ) +} + +// `for ... in` is >10x faster if the object wasn't created with `__proto__: null` +// Allocating the object is also faster, but it's only correct if the object is never +// used as an ad hoc map +// +// Only tested for v8 +// +// Usages of ` in ` can be broken, particularly when `k` isn't constant +// In this case we end up evaluating `true` if the key is on `Object.prototype` +// which isn't what we want for ad hoc maps. In these cases it's best to declare +// a null-proto object factory and use that, example: +// `const CreateObj = Object.create.bind(Object.create, null)` +// +// If initializers are needed, then we can do this: +// `const CreateObjInit = Object.setPrototypeOf.bind(Object.setPrototypeOf)` +// `CreateObjInit({ foo: 'bar' }, null)` +// +// Although using `__proto__` directly is a bit faster in this case +// + +function createSelfBound(exp: ts.Expression, ...args: ts.Expression[]) { + const bind = ts.factory.createPropertyAccessExpression(exp, 'bind') + + return ts.factory.createCallExpression(bind, undefined, [exp, ...args]) +} + +const statementsMap = new Map>() +function hoistStatement(sf: ts.SourceFile, stmt: ts.Statement) { + const statements = statementsMap.get(sf) ?? new Set() + statementsMap.set(sf, statements) + statements.add(stmt) +} + +function nullProtoFactory(ident: ts.Identifier) { + return ts.factory.createVariableStatement( + undefined, + ts.factory.createVariableDeclarationList( + [ + ts.factory.createVariableDeclaration( + ident, + undefined, + undefined, + createSelfBound( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier('Object'), + 'create' + ), + ts.factory.createNull() + ) + ), + ], + ts.NodeFlags.Const + ) + ) +} + +const didAddNullProto = new Set() +function getNullProtoFactory(sf: ts.SourceFile) { + const ident = ts.factory.createIdentifier('CreateNullProtoObject') + if (didAddNullProto.has(sf)) { + return ident + } + + didAddNullProto.add(sf) + hoistStatement(sf, nullProtoFactory(ident)) + return ident +} + +function removeProtoNull(node: ts.ObjectLiteralExpression) { + const filtered = node.properties.filter(p => !isProtoNull(p)) + if (filtered.length === node.properties.length) { + return node + } + + if (filtered.length === 0) { + return ts.factory.createCallExpression( + getNullProtoFactory(ts.getOriginalNode(node).getSourceFile()), + undefined, + [] + ) + } + + return ts.factory.updateObjectLiteralExpression( + node, + filtered, + ) +} + +function createMatchers(mappings: Mappings, extras = createExtras()): Matcher[] { + const m: Matcher[] = [] + + function getKey(k: T): NonNullable { + return { + ...mappings[k], + ...extras[k], + } as any + } + + // Custom transforms have higher priority + for (const [k, v] of Object.entries(getKey('transforms'))) { + const matchType = Array.isArray(v.matchType) ? v.matchType : [v.matchType] + if (matchType[matchType.length - 1] === ts.SyntaxKind.CallExpression || matchType[matchType.length - 1] === ts.SyntaxKind.NewExpression) { + matchType.push('expression') + } + + if (matchType[0] === ts.SyntaxKind.ForOfStatement) { + m.push({ + symbolName: v.symbolName ?? k, + // ...new () + matchType: [ts.SyntaxKind.SpreadElement, 'expression', ts.SyntaxKind.NewExpression, 'expression'], + fn: v.fn, + }) + } + + m.push({ + symbolName: v.symbolName ?? k, + matchType, + fn: v.fn, + }) + } + + // These should all be methods + for (const [k, v] of Object.entries(getKey('instanceFields'))) { + m.push({ + symbolName: k, + matchType: [ts.SyntaxKind.CallExpression, 'expression'], + fn: n => transformMethod(n, v), + }) + + if (varargsMethods.has(k)) { + m.push({ + symbolName: `${k}Apply`, + matchType: [ts.SyntaxKind.CallExpression, 'expression'], + fn: n => { + if (!ts.isCallExpression(n)) { + failOnNode('Not a call expression', n) + } + + if (ts.isArrayLiteralExpression(n.arguments[1])) { + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + n.arguments[0], + v, + ), + undefined, + n.arguments[1].elements + ) + } + + // Could also do `.apply()` + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + n.arguments[0], + v, + ), + undefined, + [ts.factory.createSpreadElement(n.arguments[1])] + ) + } + }) + } + } + + // E.g. `Reflect.apply` + for (const [k, v] of Object.entries(getKey('staticFields'))) { + const parts = v.split('.') + + m.push({ + symbolName: k, + matchType: ts.SyntaxKind.Identifier, + fn: n => { + return ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier(parts[0]), + parts[1] + ) + } + }) + + if (varargsMethods.has(k)) { + m.push({ + symbolName: `${k}Apply`, + matchType: [ts.SyntaxKind.CallExpression, 'expression'], + fn: n => { + if (!ts.isCallExpression(n)) { + failOnNode('Not a call expression', n) + } + + if (ts.isArrayLiteralExpression(n.arguments[0])) { + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier(parts[0]), + parts[1] + ), + undefined, + n.arguments[0].elements + ) + } + + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier(parts[0]), + parts[1] + ), + undefined, + [ts.factory.createSpreadElement(n.arguments[0])] + ) + } + }) + } + } + + for (const [k, v] of Object.entries(getKey('instanceGet'))) { + m.push(GetAccessorReplacer(v, k)) + } + + for (const [k, v] of Object.entries(getKey('instanceSet'))) { + m.push(SetAccessorReplacer(v, k)) + } + + return m +} + +function groupMatchers(matchers: Matcher[]) { + // Only groups 1 level deep + const m = new Map() + function getGroup(t: ts.SyntaxKind) { + if (!m.has(t)) { + m.set(t, []) + } + + return m.get(t)! + } + + for (const x of matchers) { + const mt = x.matchType + if (!Array.isArray(mt)) { + getGroup(mt).push({ ...x, matchType: [] }) + continue + } + + if (typeof mt[0] === 'string') { + throw new Error('Not expected?') + } + + getGroup(mt[0]).push({ ...x, matchType: mt.slice(1) }) + } + + return m +} + +export async function transformNodePrimordials(targets?: string[]) { + const cwd = process.cwd() + const mappingsPath = path.resolve(cwd, 'mappings.json') + const mappings: Mappings = JSON.parse(await getFs().readFile(mappingsPath, 'utf-8')) + const matchers = createMatchers(mappings) + const groups = groupMatchers(matchers) + + const targetFiles = await glob(getFs(), cwd, ['lib/**/*.js']) + const testTargets = targets?.map(s => path.resolve(cwd, 'lib', s) + '.js') + + for (const filePath of targetFiles) { + if (testTargets && testTargets.length > 0 && testTargets.indexOf(filePath) === -1) continue + + const text = await getFs().readFile(filePath, 'utf-8') + const sf = ts.createSourceFile(filePath, text, ts.ScriptTarget.ES2022, true) + const r = doMap(sf, groups) + if (r && r !== sf) { + const printer = ts.createPrinter() + await getFs().writeFile(filePath, printer.printFile(r)) + } + } +} + +function doMap(sf: ts.SourceFile, groups: Map) { + if (sf.fileName.endsWith('primordials.js')) { + return + } + + const g = createGraphCompiler({} as any, {}) + const ctx = getNullTransformationContext() + + function getUnaliasedSymbol(n: ts.Node): Symbol | undefined { + const s = g.getSymbol(n) + if (s?.declaration && ts.isBindingElement(s.declaration) && s.declaration.propertyName && ts.isIdentifier(s.declaration.propertyName)) { + ;(s as Mutable).name = s.declaration.propertyName.text + } + return s + } + + throwIfCancelled() + + function visit(node: ts.Node): ts.Node { + if (ts.isVariableStatement(node)) { + const decl = node.declarationList.declarations[0] + if (decl.initializer && ts.isIdentifier(decl.initializer) && decl.initializer.text === 'primordials') { + if (!ts.isIdentifier(decl.name)) { + if (ts.isObjectBindingPattern(decl.name)) { + for (const x of decl.name.elements) { + if (!ts.isIdentifier(x.name)) { + + } + } + } else { + + } + } + + return ts.factory.createNotEmittedStatement(node) + } + + if (ts.isIdentifier(decl.name) && decl.name.text === 'MainContextError') { + return ts.factory.createNotEmittedStatement(node) + } + } + + if (ts.isObjectLiteralExpression(node)) { + const pruned = removeProtoNull(node) + if (pruned.kind === ts.SyntaxKind.CallExpression) { + return pruned + } + return ts.visitEachChild(pruned, visit, ctx) + } + + const matchers = groups.get(node.kind) + if (!matchers) { + return ts.visitEachChild(node, visit, ctx) + } + + for (const m of matchers) { + if (isMatch(node, m, getUnaliasedSymbol)) { + return m.fn(ts.visitEachChild(node, visit, ctx)) + } + } + + return ts.visitEachChild(node, visit, ctx) + } + + const transformed = ts.visitEachChild(sf, visit, ctx) + const extraStatements = statementsMap.get(sf) + if (extraStatements) { + return ts.factory.updateSourceFile( + transformed, + [ + ...transformed.statements.slice(0, 1), // `"use strict";` + ...extraStatements, + ...transformed.statements.slice(1), + ] + ) + } + + return transformed +} + +async function parseGitTree(tree: string) { + const r = await runCommand('git', ['ls-tree', tree]) + const objects = r.split('\n').map(l => { + const [mode, type, hash, ...rest] = l.split(' ') + const name = rest.join(' ').trim() // Not sure if names can have spaces or not + + return { mode, type: type as 'blob' | 'tree', hash, name } + }) + + return objects +} + +async function catBlob(hash: string) { + const b = await runCommand('git', ['cat-file', 'blob', hash], { encoding: 'none' }) as any as Buffer + + return b +} \ No newline at end of file diff --git a/src/utils/github.ts b/src/utils/github.ts new file mode 100644 index 0000000..e452cbb --- /dev/null +++ b/src/utils/github.ts @@ -0,0 +1,110 @@ +import * as http from './http' + +export function parseDependencyRef(ref: string) { + function parsePathComponent(pathname: string) { + const [p, commitish] = pathname.split('#') + const [owner, repo] = p.replace(/\.git$/, '').split('/') + if (!repo) { + throw new Error(`Failed to parse repository from github ref: ${p}`) + } + + return { + owner, + repository: repo, + commitish: commitish as string | undefined, + } + } + + if (ref.startsWith('https://github.com')) { + return { + type: 'github', + ...parsePathComponent(ref.slice('https://github.com/'.length)) + } as const + } + + if (!ref.match(/^[a-z]+:/)) { + return { + type: 'github', + ...parsePathComponent(ref), + } as const + } + + throw new Error(`Not implemented: ${ref}`) +} + +function getToken() { + return process.env['CROSSREPO_GITHUB_TOKEN'] || process.env['GITHUB_TOKEN'] +} + +export async function fetchJson(url: string): Promise { + const token = getToken() + const authHeaders = token ? { authorization: `token ${token}` } : undefined + + return http.fetchJson(url, { + ...authHeaders, + 'user-agent': 'synapse', + accept: 'application/vnd.github+json', + }) +} + +export async function fetchData(url: string, accept?: string) { + const token = getToken() + const authHeaders = token ? { authorization: `token ${token}` } : undefined + const base = { + ...authHeaders, + 'user-agent': 'synapse', + } + + return http.fetchData(url, accept ? { ...base, accept } : base) +} + +export async function listArtifacts(owner: string, repo: string, name?: string) { + const query = name ? new URLSearchParams({ name }) : undefined + const url = `https://api.github.com/repos/${owner}/${repo}/actions/artifacts${query ? `?${query.toString()}` : ''}` + const data = await fetchJson(url) + + return data.artifacts as { id: number; name: string; created_at: string; archive_download_url: string }[] +} + +export async function downloadRepoFile(owner: string, repo: string, filePath: string, branch = 'main') { + const url = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${filePath}` + + return fetchData(url, 'application/vnd.github.v3.raw') +} + +export interface Release { + name: string + tag_name: string + published_at: string + assets: { + name: string + content_type: string + browser_download_url: string + }[] +} + +export async function listReleases(owner: string, repo: string): Promise { + return fetchJson(`https://api.github.com/repos/${owner}/${repo}/releases`) +} + +async function getLatestRelease(owner: string, repo: string): Promise { + return fetchJson(`https://api.github.com/repos/${owner}/${repo}/releases/latest`) +} + +export async function getRelease(owner: string, repo: string, tagName?: string): Promise { + if (!tagName) { + return getLatestRelease(owner, repo) + } + + return fetchJson(`https://api.github.com/repos/${owner}/${repo}/releases/tags/${tagName}`) +} + +export async function downloadAssetFromRelease(owner: string, repo: string, name: string, tagName?: string): Promise { + const release = await getRelease(owner, repo, tagName) + const asset = release.assets.find(a => a.name === name) + if (!asset) { + throw new Error(`Asset not found: ${name} [release: ${release.tag_name}]`) + } + + return fetchData(asset.browser_download_url) +} diff --git a/src/utils/glob.ts b/src/utils/glob.ts new file mode 100644 index 0000000..8d7431a --- /dev/null +++ b/src/utils/glob.ts @@ -0,0 +1,495 @@ +import * as path from 'node:path' +import { Fs } from '../system' +import { keyedMemoize } from '../utils' +import { getLogger } from '..' + +interface Wildcard { + readonly type: 'wildcard' + readonly value: '*' | '?' | '**' +} + +interface Literal { + readonly type: 'literal' + readonly value: string +} + +interface Separator { + readonly type: 'sep' +} + +interface CharacterSet { + readonly type: 'set' + readonly values: (string | [string, string])[] + readonly negate?: boolean +} + +type GlobComponent = + | Literal + | Separator + | Wildcard + | CharacterSet + +enum ParseState { + Initial, + Set, +} + +// https://www.man7.org/linux/man-pages/man7/glob.7.html +function parseGlobPattern(pattern: string) { + let state = ParseState.Initial + let negate = false + let charSet: (string | [string, string])[] = [] + + const components: GlobComponent[] = [] + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i] + switch (state) { + case ParseState.Initial: + switch (c) { + case '*': + if (pattern[i + 1] === '*') { + components.push({ type: 'wildcard', value: '**' }) + i += 1 + break + } + if (pattern[i + 1] === '?') { + let j = i + 2 + while (j < pattern.length) { + if (pattern[j] !== '?') { + break + } + j += 1 + } + components.push({ type: 'wildcard', value: '*' }) + i += j - i - 1 + break + } + case '?': + components.push({ type: 'wildcard', value: c }) + break + case '/': + components.push({ type: 'sep' }) + break + case '[': + state = ParseState.Set + if (pattern[i + 1] === '!') { + negate = true + i += 1 + } + break + default: + let j = i + 1 + while (j < pattern.length) { + if (pattern[j] === '*' || pattern[j] === '?' || pattern[j] === '/' || pattern[j] === '[') { + break + } + j += 1 + } + components.push({ type: 'literal', value: pattern.slice(i, j) }) + i += j - i - 1 + break + } + break + case ParseState.Set: + switch (c) { + case ']': + if (charSet.length > 0) { + components.push({ type: 'set', values: charSet, negate }) + charSet = [] + negate = false + state = ParseState.Initial + } else { + charSet.push(c) + } + break + case '-': + if (charSet.length === 0 || pattern[i + 1] === ']') { + charSet.push(c) + } else { + i += 1 + const previous = charSet.pop()! + if (typeof previous !== 'string') { + throw new Error(`Invalid character range at position: ${i}`) + } + charSet.push([previous, pattern[i + 1]]) + } + break + default: + charSet.push(c) + break + } + } + } + + if (components.length === 0) { + throw new Error(`Bad parse: empty glob pattern`) + } + + if (state !== ParseState.Initial) { + throw new Error(`Bad parse: unfinished character set`) + } + + return components +} + +function splitComponents(components: GlobComponent[]) { + const groups: Exclude[][] = [] + let cg: Exclude[] = groups[0] = [] + for (const c of components) { + if (c.type === 'sep') { + cg = groups[groups.length] = [] + } else { + cg.push(c) + } + } + return groups +} + +const getCharCode = (v: string, caseInsensitive: boolean) => (caseInsensitive ? v.toLowerCase() : v).charCodeAt(0) + +function matchCharOrRange(char: string, value: string | string[], caseInsensitive: boolean) { + if (typeof value === 'string') { + return caseInsensitive ? char.toLowerCase() === value.toLowerCase() : char === value + } + + const c = getCharCode(char, caseInsensitive) + const x = getCharCode(value[0], caseInsensitive) + const y = getCharCode(value[1], caseInsensitive) + + return c >= x && c <= y +} + +function matchSet(char: string, s: CharacterSet, caseInsensitive = false) { + for (const v of s.values) { + const matched = matchCharOrRange(char, v, caseInsensitive) + if ((s.negate && !matched) || (!s.negate && matched)) { + return true + } + } + return false +} + +function matchComponent(segment: string, pattern: Exclude[], matchHidden = false, caseInsensitive = false): boolean { + if (segment[0] === '.' && !matchHidden && pattern[0].type === 'wildcard') { + return false + } + + let i = 0 + let j = 0 + for (j = 0; j < pattern.length; j++) { + const current = pattern[j] + switch (current.type) { + case 'set': + if (!matchSet(segment[i], current)) { + return false + } + i += 1 + break + case 'literal': + if (current.value !== segment.slice(i, i + current.value.length)) { + return false + } + i += current.value.length + break + case 'wildcard': + switch (current.value) { + case '?': + i += 1 + break + case '*': + while (j < pattern.length && pattern[j].type === 'wildcard') j++ + if (j === pattern.length) { + return true + } + + const n = pattern[j] + switch (n.type) { + case 'wildcard': + throw new Error('Bad state') + case 'literal': + const ni = segment.indexOf(n.value, i) + if (ni === -1) { + return false + } + i = ni + n.value.length + break + case 'set': + let matched = false + while (i < segment.length) { + if (matchSet(segment[i], n)) { + matched = true + i += 1 + break + } + i += 1 + } + if (!matched) { + return false + } + break + } + break + case '**': + return true + } + break + } + } + + return segment.length === i && pattern.length === j +} + +export type GlobHost = Pick + +interface PatternOptions { + readonly exclude?: boolean + readonly matchHidden?: boolean // Defaults to `true` when using an exclude pattern + readonly caseInsensitive?: boolean +} + +async function multiGlob(fs: GlobHost, dir: string, patterns: Exclude[][][], options = new Map()): Promise { + const res: string[] = [] + if (patterns.length === 0) { + return res + } + + if ([...options.values()].every(x => x.exclude) && options.size === patterns.length) { + return res + } + + function match(index: number, name: string, pattern: Exclude[]) { + const opt = options.get(index) + const matchHidden = opt?.matchHidden ?? !!opt?.exclude + + return matchComponent(name, pattern, matchHidden, opt?.caseInsensitive) + } + + const literalSegments: [index: number, group: Literal[]][] = [] + const wildcardSegments: [index: number, group: Exclude[]][] = [] + const globstars = new Set() + + for (let i = 0; i < patterns.length; i++) { + const groups = patterns[i] + const g = groups.shift() + if (!g || g.length === 0) { + continue + } + + const isGlobstar = g[0].type === 'wildcard' && g[0].value === '**' + const isWildcard = isGlobstar || g.some(x => x.type !== 'literal') // Case insentive -> wildcard? + if (isWildcard) { + if (isGlobstar) { + globstars.add(i) + } + wildcardSegments.push([i, g]) + } else { + literalSegments.push([i, g as Literal[]]) + } + } + + // Sorting + const isExcluded = (index: number) => !!options.get(index)?.exclude + const sortExcludedFirst = (a: number, b: number) => { + const c = isExcluded(a) + const d = isExcluded(b) + + return c === d ? 0 : c ? -1 : 1 + } + + wildcardSegments.sort((a, b) => sortExcludedFirst(a[0], b[0])) + literalSegments.sort((a, b) => sortExcludedFirst(a[0], b[0])) + // + + const matchedFiles = new Set() + const matchedDirectoriesAll = new Map() + const matchedDirectories = new Map() + + const getStats = keyedMemoize((fileName: string) => fs.stat(fileName).catch(e => { + if ((e as any).code !== 'ENOENT') { + throw e + } + })) + + function matchFile(index: number, name: string, filePath: string) { + const included = !options.get(index)?.exclude + if (included) { + res.push(filePath) + } + matchedFiles.add(name) + } + + for (let i = 0; i < literalSegments.length; i++) { + const [j, g] = literalSegments[i] + const literal = g.map(c => c.value).join('') + if (matchedFiles.has(literal) || matchedDirectoriesAll.get(literal) === false) { + continue + } + + const p = path.join(dir, literal) + const stats = await getStats(p) // could call this in parallel + if (!stats) { + continue + } + + if (stats.type === 'directory') { + if (patterns[j].length > 0) { + matchedDirectories.set(literal, [j, ...(matchedDirectories.get(literal) ?? [])]) + } else { + matchedDirectoriesAll.set(literal, !options.get(j)?.exclude) + } + } else if (stats.type === 'file') { + matchFile(j, literal, p) + } + } + + function searchDir(name: string) { + if (matchedDirectoriesAll.get(name) === false) { + return + } + + const nextPatterns: number[] = [] + for (let i = 0; i < wildcardSegments.length; i++) { + const [j, g] = wildcardSegments[i] + + if (globstars.has(j)) { + if (match(j, name, g)) { + nextPatterns.push(j) + } + + if (patterns[j].length === 1 && match(j, name, patterns[j][0])) { + const included = !options.get(j)?.exclude + matchedDirectoriesAll.set(name, included) + if (!included) { + return + } + } + } else if (match(j, name, g)) { + if (patterns[j].length > 0) { + nextPatterns.push(j) + } else { + const included = !options.get(j)?.exclude + matchedDirectoriesAll.set(name, included) + if (!included) { + return + } + } + } + } + + if (matchedDirectories.has(name)) { + nextPatterns.push(...matchedDirectories.get(name)!) + } + + if (nextPatterns.length > 0) { + matchedDirectories.set(name, nextPatterns) + } + } + + if (wildcardSegments.length > 0) { + const files = await fs.readDirectory(dir) + for (const f of files) { + if (f.type === 'directory') { + searchDir(f.name) + } else if (f.type === 'file' && !matchedFiles.has(f.name)) { + for (let i = 0; i < wildcardSegments.length; i++) { + const [j, g] = wildcardSegments[i] + + if (globstars.has(j)) { + if (patterns[j].length === 1 && match(j, f.name, patterns[j][0])) { + matchFile(j, f.name, path.join(dir, f.name)) + break + } + } else if (patterns[j].length === 0) { + if (match(j, f.name, g)) { + matchFile(j, f.name, path.join(dir, f.name)) + break + } + } + } + } + } + } + + async function readAll(name: string) { + // TODO: if `matchHidden` then use `*` + const nextPatterns: typeof patterns = [splitComponents(parseGlobPattern('**/[!.]*'))] + const nextOptions: typeof options = new Map() + + for (const i of matchedDirectories.get(name) ?? []) { + const opt = options.get(i) + if (!opt?.exclude && !(patterns[i][0][0].type === 'literal' && (patterns[i][0][0]as any).value.startsWith('.'))) { // FIXME: is this right? + continue + } + + // TODO: dedupe code w/ the `matchedDirectories` loop + nextOptions.set(nextPatterns.length, opt ?? {}) + if (globstars.has(i)) { + nextPatterns.push([[{ type: 'wildcard', value: '**' }], ...patterns[i]]) + nextOptions.set(nextPatterns.length, opt ?? {}) + nextPatterns.push([...patterns[i]]) + } else { + nextPatterns.push([...patterns[i]]) + } + } + + return multiGlob(fs, path.join(dir, name), nextPatterns, nextOptions) + } + + const promises: Promise[] = [] + + for (const [name, included] of matchedDirectoriesAll.entries()) { + if (included) { + promises.push(readAll(name)) + } + } + + for (const [name, indices] of matchedDirectories.entries()) { + if (matchedDirectoriesAll.has(name)) { + continue + } + + const nextPatterns: typeof patterns = [] + const nextOptions: typeof options = new Map() + for (const i of indices) { + const opt = options.get(i) ?? {} + nextOptions.set(nextPatterns.length, opt) + + if (globstars.has(i)) { + nextPatterns.push([[{ type: 'wildcard', value: '**' }], ...patterns[i]]) + nextOptions.set(nextPatterns.length, opt) + nextPatterns.push([...patterns[i]]) + } else { + nextPatterns.push([...patterns[i]]) + } + } + + promises.push(multiGlob(fs, path.join(dir, name), nextPatterns, nextOptions)) + } + + for (const arr of await Promise.all(promises)) { + res.push(...arr) + } + + return res +} + +export function glob(fs: GlobHost, dir: string, include: string[], exclude: string[] = []) { + const patterns = include.map(parseGlobPattern).map(splitComponents) + const options = new Map() + for (const p of exclude) { + options.set(patterns.length, { exclude: true }) + + // We implicitly globstar things that might be directories + if (!p.startsWith('**/') && !p.includes('/') && !p.includes('*') && (!p.includes('.') || p.startsWith('.'))) { + patterns.push(splitComponents(parseGlobPattern(`**/${p}`))) + } else { + patterns.push(splitComponents(parseGlobPattern(p))) + } + } + + return multiGlob(fs, dir, patterns, options) +} + +// TODO: tests +// describe('...') +// \ No newline at end of file diff --git a/src/utils/http.ts b/src/utils/http.ts new file mode 100644 index 0000000..071cec3 --- /dev/null +++ b/src/utils/http.ts @@ -0,0 +1,201 @@ +import * as zlib from 'node:zlib' +import { getLogger } from '../logging' + +// Only covers happy paths +interface DownloadProgressListener { + onStart: (path: string, size: number) => void + onProgress: (path: string, bytes: number) => void + onEnd: (path: string) => void +} + +export function createRequester(baseUrl: string, listener?: DownloadProgressListener) { + let idCounter = 0 + const queued: [id: number, fn: () => Promise, resolve: any, reject: any, retryCount: number][] = [] + const pending = new Map>() + const maxConnections = 25 + + const https = require('node:https') as typeof import('node:https') + + const agent = new https.Agent({ + keepAlive: true, + }) + + function run(fn: () => Promise, id: number, retryCount: number) { + const promise = fn() + .catch(err => { + // FIXME: should throw on most 4xx + if ((err as any).statusCode === 400 || (err as any).statusCode === 401 || (err as any).statusCode === 403 || (err as any).statusCode === 404) { + throw err + } + + let retryDelay = 250 * (retryCount + 1) + if ((err as any).statusCode === 429) { + const retryAfter = (err as any).headers['retry-after'] + const n = Number(retryAfter) + if (!isNaN(n)) { + getLogger().log(`Retrying after ${n} seconds`) + retryDelay = n * 1000 + } else if (retryAfter) { + const d = new Date(retryAfter) + retryDelay = d.getTime() - Date.now() + getLogger().log(`Retrying after ${retryDelay} ms`) + } + } + if (retryCount > 3) { + throw err + } + + getLogger().warn(`Retrying download due to error`, err) + + return new Promise(r => setTimeout(r, retryDelay)).then(() => enqueue(fn, id, retryCount + 1)) + }) + .finally(() => complete(id)) + + pending.set(id, promise) + + return promise + } + + function complete(id: number) { + pending.delete(id) + while (queued.length > 0 && pending.size < maxConnections) { + const [nid, nfn, resolve, reject, retryCount] = queued.shift()! + run(nfn, nid, retryCount).then(resolve, reject) + } + } + + function enqueue(fn: () => Promise, id = idCounter++, retryCount = 0): Promise { + if (pending.size >= maxConnections) { + return new Promise((resolve, reject) => { + queued.push([id, fn, resolve, reject, retryCount]) + }) + } else { + return run(fn, id, retryCount) + } + } + + function request(route: string, body?: any, unzip = false, opt?: { etag?: string; maxAge?: number; acceptGzip?: boolean, abortController?: AbortController; headers?: Record; alwaysJson?: boolean }) { + const [method, path] = route.split(' ') + const url = new URL(path, baseUrl) + + opt?.abortController?.signal.throwIfAborted() + + const doReq = () => new Promise((resolve, reject) => { + const headers: Record = typeof body === 'object' + ? { 'content-type': 'application/json', ...opt?.headers } + : opt?.headers ?? {} + + headers['user-agent'] = 'synapse' + + if (opt?.etag) { + headers['if-none-match'] = opt.etag + } + + if (opt?.acceptGzip) { + headers['accept-encoding'] = 'gzip' + } + + const req = https.request(url, { method, headers, agent, signal: opt?.abortController?.signal }, resp => { + const buffer: any[] = [] + const contentLength = resp.headers['content-length'] + if (listener && contentLength) { + listener.onStart(path, Number(contentLength)) + } + + const unzipStream = (unzip || resp.headers['content-encoding'] === 'gzip') ? zlib.createGunzip() : undefined + + unzipStream?.on('data', d => buffer.push(d)) + unzipStream?.on('end', () => { + if (resp.headers['content-type'] === 'application/json') { + resolve(JSON.parse(buffer.join(''))) + } else { + resolve(Buffer.concat(buffer)) + } + }) + resp.on('data', d => { + if (unzipStream) { + unzipStream.write(d) + } else { + buffer.push(d) + } + + if (listener && contentLength) { + listener.onProgress(path, d.length) + } + }) + resp.on('end', () => { + if (!resp.statusCode) { + return reject(new Error('Response contained no status code')) + } + + if (resp.statusCode >= 400) { + return reject(Object.assign(new Error(buffer.join('')), { + headers: resp.headers, + statusCode: resp.statusCode, + })) + } + + if (resp.statusCode === 302) { + return request(`${method} ${resp.headers['location']}`, undefined, undefined, opt).then(resolve, reject) + } + + if (opt) { + opt.etag = resp.headers['etag'] + + if (resp.headers['cache-control']) { + const directives = resp.headers['cache-control'].split(',').map(x => x.trim()) + const maxAge = directives.find(d => d.startsWith('max-age=')) + if (maxAge) { + const seconds = Number(maxAge.slice('max-age='.length)) + const age = Number(resp.headers['age'] || 0) + opt.maxAge = seconds - age + } + } + } + + if (resp.statusCode === 304) { + return resolve(undefined) + } + + if (listener && contentLength) { + listener.onEnd(path) + } + + if (unzipStream) { + unzipStream.end() + } else if (resp.headers['content-type'] === 'application/json' || opt?.alwaysJson) { + resolve(JSON.parse(buffer.join(''))) + } else { + resolve(Buffer.concat(buffer)) + } + }) + resp.on('error', reject) + }) + + if (typeof body === 'object') { + req.end(JSON.stringify(body)) + } else { + req.end(body) + } + }) + + return enqueue(doReq) + } + + return request +} + +export function fetchJson(url: string, headers?: Record): Promise { + const parsedUrl = new URL(url) + const request = createRequester(parsedUrl.origin) + + // this is bad software. i'm making bad software + return request(`GET ${parsedUrl.pathname}`, undefined, undefined, { headers, alwaysJson: true }) +} + +export function fetchData(url: string, headers?: Record): Promise { + const parsedUrl = new URL(url) + const request = createRequester(parsedUrl.origin) + + return request(`GET ${parsedUrl.pathname}`, undefined, undefined, { headers, acceptGzip: true }) +} \ No newline at end of file diff --git a/src/utils/memento.ts b/src/utils/memento.ts new file mode 100644 index 0000000..5c75e4e --- /dev/null +++ b/src/utils/memento.ts @@ -0,0 +1,49 @@ +import * as path from 'node:path' +import { Fs } from '../system' +import { throwIfNotFileNotFoundError } from '../utils' + +// Roughly inspired by VS Code's "Memento" API + +export interface Memento { + get(key: string): Promise + get(key: string, defaultValue: T): Promise + set(key: string, value: T): Promise + delete(key: string): Promise +} + +export function createMemento(fs: Pick, dir: string): Memento { + const getLocation = (key: string) => path.resolve(dir, key) + + async function get(key: string, defaultValue?: T): Promise { + try { + return JSON.parse(await fs.readFile(getLocation(key), 'utf-8')) + } catch (e) { + throwIfNotFileNotFoundError(e) + + // Delete data when JSON is malformed? + return defaultValue + } + } + + async function set(key: string, value: T) { + await fs.writeFile(getLocation(key), JSON.stringify(value)) + } + + async function _delete(key: string) { + await fs.deleteFile(getLocation(key)).catch(throwIfNotFileNotFoundError) + } + + return { get, set, delete: _delete } +} + +export interface TypedMemento { + get(): Promise + set(value: T): Promise +} + +export function createTypedMemento(memento: Memento, key: string): TypedMemento { + return { + get: () => memento.get(key), + set: value => memento.set(key, value), + } +} diff --git a/src/utils/process.ts b/src/utils/process.ts new file mode 100644 index 0000000..17eb3d7 --- /dev/null +++ b/src/utils/process.ts @@ -0,0 +1,127 @@ +import * as child_process from 'node:child_process' + +interface RunCommandOptions extends child_process.SpawnOptions { + readonly input?: string | Uint8Array // Passed directly to stdin + /** Defaults to `utf-8` */ + readonly encoding?: BufferEncoding | 'none' +} + +export function runCommand(executable: string, args: string[], opts?: RunCommandOptions): Promise +export function runCommand(executable: string, args: string[], opts: RunCommandOptions & { encoding: 'none' }): Promise +export function runCommand(executable: string, args: string[], opts: RunCommandOptions = {}) { + const runner = createCommandRunner(opts) + + return runner(executable, args) as Promise +} + +export function runCommandStdErr(executable: string, args: string[], opts?: RunCommandOptions): Promise +export function runCommandStdErr(executable: string, args: string[], opts: RunCommandOptions & { encoding: 'none' }): Promise +export async function runCommandStdErr(executable: string, args: string[], opts: RunCommandOptions = {}) { + const proc = child_process.spawn(executable, args, { ...opts }) + const result = await toPromise(proc) + + return result.stderr +} + +export function execCommand(cmd: string, opts: child_process.ExecOptions = {}) { + const runner = createCommandRunner(opts) + + return runner(cmd) +} + +export function createCommandRunner(opts?: RunCommandOptions): (cmd: string, args?: string[]) => Promise +export function createCommandRunner(opts: RunCommandOptions & { encoding: 'none' }): (cmd: string, args?: string[]) => Promise +export function createCommandRunner(opts: RunCommandOptions = {}) { + async function runCommand(executableOrCommand: string, args?: string[]): Promise { + const proc = !args + ? child_process.spawn(executableOrCommand, { shell: true, ...opts }) + : child_process.spawn(executableOrCommand, args, { shell: true, ...opts }) + + // Likely not needed + if (proc.exitCode || proc.signalCode) { + throw new Error(`Non-zero exit code: ${proc.exitCode} [signal ${proc.signalCode}]`) + } + + if (opts.input) { + proc.stdin?.end(opts.input) + } + + const result = await toPromise(proc, opts?.encoding === 'none' ? undefined : (opts.encoding ?? 'utf-8')) + + return result.stdout + } + + return runCommand +} + +function toPromise(proc: child_process.ChildProcess, encoding?: BufferEncoding) { + const stdout: any[] = [] + const stderr: any[] = [] + proc.stdout?.on('data', chunk => stdout.push(chunk)) + proc.stderr?.on('data', chunk => stderr.push(chunk)) + + // XXX: needed to capture original trace + const _err = new Error() + + function getResult(chunks: any[], enc = encoding) { + const buf = Buffer.concat(chunks) + + return enc ? buf.toString(enc) : buf + } + + const p = new Promise<{ stdout: string | Buffer; stderr: string | Buffer}>((resolve, reject) => { + function onError(e: unknown) { + reject(e) + proc.kill() + } + + proc.on('error', onError) + proc.on('close', (code, signal) => { + if (code !== 0) { + const message = `Non-zero exit code: ${code} [signal ${signal}]` + const err = Object.assign( + new Error(message), + { code, stdout: getResult(stdout), stderr: getResult(stderr, 'utf-8') }, + { stack: message + '\n' + _err.stack?.split('\n').slice(1).join('\n') } + ) + + reject(err) + } else { + resolve({ stdout: getResult(stdout), stderr: getResult(stderr) }) + } + }) + + // Likely not needed + if (!proc.stdout && !proc.stderr) { + proc.on('exit', (code, signal) => { + if (code !== 0) { + const err = Object.assign( + new Error(`Non-zero exit code: ${code} [signal ${signal}]`), + { code, stdout: '', stderr: '' } + ) + + reject(err) + } else { + resolve({ stdout: '', stderr: '' }) + } + }) + } + }) + + return p +} + +export function patchPath(dir: string, env = process.env) { + return { + ...env, + PATH: `${dir}${env.PATH ? `:${env.PATH}` : ''}`, + } +} + +export async function which(executable: string) { + if (process.platform === 'win32') { + return runCommand('where', [executable]) + } + + return runCommand('which', [executable]) +} diff --git a/src/utils/tar.ts b/src/utils/tar.ts new file mode 100644 index 0000000..60522c2 --- /dev/null +++ b/src/utils/tar.ts @@ -0,0 +1,201 @@ +import * as path from 'node:path' +import { runCommand } from './process' +import { ensureDir, memoize } from '../utils' +import { getFs } from '../execution' +import { randomUUID } from 'node:crypto' + +// Pre-posix.1-1988 tarball header format (field size in bytes): +// File name - 100 +// File mode - 8 (octal) +// Owner id - 8 (octal) +// Group id - 8 (octal) +// File size - 12 (octal) +// * Highest order bit set to 1 means it's encoded as base-256 instead of octal +// Last modified in Unix time - 12 (octal) +// Checksum for header - 8 +// Link indicator - 1 +// * 0 - normal file +// * 1 - hard link +// * 2 - symbolic link +// * 3 - character special (UStar) +// * 4 - block special (UStar) +// * 5 - directory (UStar) +// * 6 - FIFO (UStar) +// * 7 - contiguous file (UStar) + +// Name of linked file - 100 + +// Anything marked "octal" means it's stored as ASCII text +// Headers are padded to 512 bytes + +// UStar header format is an extended form +// Check for "ustar" at offset 257 +// If present, there may be a filename prefix at offset 345 (155 bytes) + +export interface TarballFile { + path: string + mode: number + contents: Buffer + // mtime: number + // uid: number + // gid: number +} + +const toString = (buf: Buffer, start: number, length: number) => + String.fromCharCode(...buf.subarray(start, start + length)).split('\0', 2)[0] + +export function extractTarball(buf: Buffer): TarballFile[] { + let i = 0 + const files: TarballFile[] = [] + while (i < buf.length) { + const isUstar = toString(buf, i + 257, 6) === 'ustar' + const prefix = isUstar ? toString(buf, i + 345, 155) : undefined + + const name = toString(buf, i, 100) + const mode = parseInt(toString(buf, i + 100, 8), 8) + const size = parseInt(toString(buf, i + 124, 12), 8) + i += 512 + + if (!isNaN(size)) { + const contents = buf.subarray(i, i + size) + const p = prefix ? path.join(prefix, name) : name + + files.push({ + path: p, + mode, + contents, + }) + + // File data section is always padded to the nearest 512 byte increment + i += (size + 511) & ~511 + } + } + + return validateTarball(files) +} + +function validateTarball(files: TarballFile[]) { + for (const f of files) { + if (path.isAbsolute(f.path)) { + throw new Error(`Found absolute file paths: ${f.path}`) + } + if (f.path.split('/').some(s => s === '..')) { + throw new Error(`Found relative file path: ${f.path}`) + } + } + return files +} + +const toOctal = (n: number, size: number) => n.toString(8).padStart(size - 1, '0') + '\0' + +export function createTarball(files: TarballFile[]): Buffer { + validateTarball(files) + + const totalSize = files.map(f => ((f.contents.length + 511) & ~511) + 512).reduce((a, b) => a + b, 0) + const buf = Buffer.alloc(totalSize) + + let i = 0 + for (const f of files) { + const uid = 0 + const gid = 0 + const size = f.contents.length + const mtime = 0 + + let relPath + if (f.path.length >= 100) { + buf.write('ustar', i + 257, 6, 'ascii') + + const sepIndex = f.path.indexOf(path.sep, f.path.length - 100) + if (sepIndex === -1) { + throw new Error(`Failed to find path separator: ${f.path}`) + } + + relPath = f.path.slice(sepIndex+1) + buf.write(f.path.slice(0, sepIndex), i + 345, 155, 'ascii') + } + + buf.write(relPath ?? f.path, i, 100, 'ascii') + buf.write(toOctal(f.mode, 8), i + 100, 8, 'ascii') + buf.write(toOctal(uid, 8), i + 108, 8, 'ascii') + buf.write(toOctal(gid, 8), i + 116, 8, 'ascii') + buf.write(toOctal(size, 12), i + 124, 12, 'ascii') + buf.write(toOctal(mtime, 12), i + 136, 12, 'ascii') + + + buf.write(' '.repeat(8), i + 148, 8, 'ascii') + let checksum = 0 + for (let j = 0; j < 512; j++) { + checksum += buf[i + j] + } + + buf.write((checksum & 0o777777).toString(8).padStart(6, '0') + '\0 ', i + 148, 8, 'ascii') + i += 512 + buf.set(f.contents, i) + i += (size + 511) & ~511 + } + + return buf +} + + +export async function extractToDir(data: Buffer, dir: string, ext: '.xz' | '.zip', stripComponents = 1) { + await ensureDir(dir) + + const args: string[] = [] + if (stripComponents !== 0) { + args.push(`--strip-components=${stripComponents}`) + } + + switch (ext) { + case '.xz': + args.unshift('-xJf-') + break + case '.zip': + args.unshift('-xzf-') + break + default: + throw new Error(`Unknown extname: ${ext}`) + } + + await runCommand('tar', args, { cwd: dir, input: data }) +} + +export const hasBsdTar = memoize(async () => { + const res = await runCommand('tar', ['--version']) + + return res.includes('bsdtar') +}) + +export async function extractFileFromZip(zip: Buffer, fileName: string) { + if (!(await hasBsdTar())) { + const tmp = path.resolve(process.cwd(), 'dist', `tmp-${randomUUID()}.zip`) + await getFs().writeFile(tmp, zip) + const res = await runCommand('unzip', ['-p', tmp, fileName], { + encoding: 'none', + }).finally(async () => { + await getFs().deleteFile(tmp) + }) + + return res as any as Buffer + } + + if (process.platform === 'win32') { + const tmp = path.resolve(process.cwd(), 'dist', `tmp-${randomUUID()}.zip`) + await getFs().writeFile(tmp, zip) + await runCommand('tar', ['-zxf', tmp, fileName], { + encoding: 'none', + }).finally(async () => { + await getFs().deleteFile(tmp) + }) + + return getFs().readFile(fileName) + } + + // Only works with `bsdtar` + const res = await runCommand('tar', ['-zxf-', '-O', fileName], { + input: zip, + encoding: 'none', + }) + + return res as any as Buffer +} diff --git a/src/workspaces.ts b/src/workspaces.ts new file mode 100644 index 0000000..da51e0b --- /dev/null +++ b/src/workspaces.ts @@ -0,0 +1,967 @@ +import * as os from 'node:os' +import * as path from 'node:path' +import * as fs from 'node:fs/promises' +import { Fs, SyncFs } from './system' +import { DeployOptions } from './deploy/deployment' +import { Remote, findRepositoryDir, getCurrentBranch, getCurrentBranchSync, listRemotes } from './git' +import { getLogger } from './logging' +import { keyedMemoize, memoize, throwIfNotFileNotFoundError, tryReadJson, tryReadJsonSync } from './utils' +import { glob } from './utils/glob' +import { getBuildTarget, getBuildTargetOrThrow, getFs, isInContext } from './execution' +import { getBackendClient } from './backendClient' +import { projects, processes } from '@cohesible/resources' +import { getPackageJson } from './pm/packageJson' +import { randomUUID } from 'node:crypto' + +// Workspaces are state + source code!!! +// A workspace is directly tied to a source control repo +// State is isolated per-branch. Each branch can be thought as its own +// independent version of the app. +// +// "Merging" branches _never_ merges state, only source code. State is updated +// using the new source code. Failures to update the state result in a rollback. + +// IMPORTANT: +// need to implement this +// https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + +export interface Workspace { + readonly id: string + readonly defaultBranch?: string +} + +export interface LocalWorkspace extends Workspace { + readonly directory: string + readonly currentBranch: string + readonly buildDirectory?: string + readonly deployOptions?: DeployOptions +} + +export type SynapseConfiguration = DeployOptions + +const synDirName = '.synapse' + +// We should only follow this spec for Linux/macOS +// TODO: what about Windows? +// +// XDG_DATA_HOME - defaults to $HOME/.local/share +// XDG_CONFIG_HOME - defaults to $HOME/.config +// XDG_STATE_HOME - for persisent but not super important data e.g. logs, defaults to $HOME/.local/state +// XDG_CACHE_HOME - defaults to $HOME/.cache +// XDG_RUNTIME_DIR - sockets, named pipes, must have o700 perms +// - no explicit fallback specified +// * User-specific executable files may be stored in $HOME/.local/bin. +// * If an implementation encounters a relative path in any of these variables it should consider the path invalid and ignore it. +// +// const shouldUseXdg = true + +export function getUserSynapseDirectory() { + return process.env['SYNAPSE_INSTALL'] ?? path.resolve(os.homedir(), synDirName) +} + +export function getPackageCacheDirectory() { + return path.resolve(getGlobalCacheDirectory(), 'packages') +} + +export function getLinkedPackagesDirectory() { + return path.resolve(getPackageCacheDirectory(), 'linked') +} + +export function getLogsDirectory() { + return path.resolve(getUserSynapseDirectory(), 'logs') +} + +export function getDeploymentBuildDirectory(buildTarget: Pick) { + return path.resolve(buildTarget.buildDir, 'deployments', buildTarget.deploymentId!) +} + +export function getGlobalCacheDirectory() { + return path.resolve(getUserSynapseDirectory(), 'cache') +} + +export function getV8CacheDirectory() { + return path.resolve(getGlobalCacheDirectory(), 'v8') +} + +export function getProviderTypesDirectory(workingDirectory: string) { + return path.resolve(workingDirectory, 'node_modules', '@types', 'synapse-providers') +} + +export function getProviderCacheDir() { + return path.resolve(getGlobalCacheDirectory(), 'providers') +} + +export function getBinDirectory() { + return path.resolve(getUserSynapseDirectory(), 'bin') +} + +export function getGitDirectory() { + return path.resolve(getUserSynapseDirectory(), 'git') +} + +export function getUserEnvFileName() { + return path.resolve(getUserSynapseDirectory(), 'env') +} + +export function getToolsDirectory() { + return path.resolve(getUserSynapseDirectory(), 'tools') +} + +export function getSocketsDirectory() { + return path.resolve(getUserSynapseDirectory(), 'sockets') +} + +export function getUserConfigFilePath() { + return path.resolve(getUserSynapseDirectory(), 'config.json') +} + +export interface Project { + readonly id: string + readonly name: string + readonly apps?: Record + readonly programs?: Record + + // This associates packages with a particular program + readonly packages?: Record +} + +export interface Program { + readonly id: string + readonly name?: string + readonly workingDirectory?: string + readonly deployOptions?: any + readonly branch?: string +} + +export interface Deployment { + readonly id: string + readonly name?: string + readonly program: Program['id'] + readonly project: Project['id'] +} + +interface Environment { + readonly name: string + readonly config?: Record +} + +interface BuildTargetOptions { + // readonly branch?: string + readonly project?: Project['name'] | Project['id'] + readonly program?: Program['name'] | Program['id'] + readonly deployment?: Deployment['name'] | Deployment['id'] + readonly environmentName?: Environment['name'] +} + +const getCurrentBranchCached = keyedMemoize(getCurrentBranch) + +async function getDeploymentById(id: string) { + const ents = await getEntities() + const deployment = ents.deployments[id] + if (!deployment) { + throw new Error(`No deployment found: ${id}`) + } + + return deployment +} + +async function getProjectDirectory(id: string) { + const ents = await getEntities() + const rootDirectory = ents.projects[id]?.directory + if (!rootDirectory) { + throw new Error(`Missing project: ${id}`) + } + + return rootDirectory +} + +function getProjectDirectorySync(id: string) { + const ents = getEntitiesSync() + const rootDirectory = ents.projects[id]?.directory + if (!rootDirectory) { + throw new Error(`Missing project: ${id}`) + } + + return rootDirectory +} + +async function tryInitProject(cwd: string) { + const gitRepo = await findRepositoryDir(cwd) + if (!gitRepo) { + return + } + + const remotes = await listRemotes(gitRepo) + if (remotes.length === 0) { + return + } + + return initProject(gitRepo, remotes) +} + +async function findProject(cwd: string) { + const found = await findProjectFromDir(cwd) + if (!found) { + return + } + + return { + ...found, + rootDir: found.directory + } +} + +// TODO: we should hash all program IDs to make them safe to use as filenames and store metadata somewhere else +function getProgramIdNoPkg(workingDirectory: string, rootDir: string) { + const base = path.relative(rootDir, workingDirectory) + if (!base) { + return 'dir___root' + } + + return `dir__${base.replaceAll(path.sep, '_')}` +} + +async function findProgram(cwd: string, rootDir: string, projectId: string) { + const relPath = path.relative(rootDir, cwd) + if (relPath.startsWith('..')) { + throw new Error(`Current directory "${cwd}" is not inside project directory: ${rootDir}`) + } + + const pkg = await getPackageJson(getFs(), cwd, false) + const workingDirectory = pkg?.directory ?? cwd + const name = pkg?.data.name ?? getProgramIdNoPkg(workingDirectory, rootDir) + const branch = await getCurrentBranchCached(rootDir) + const programId = branch ? `${branch}_${name}` : name + const relativeWorkingDir = cwd === rootDir ? undefined : relPath + + const state = await getProjectState(projectId) + if (state) { + const existing = state.programs[programId] + if (!existing) { + getLogger().debug(`Initialized program "${programId}"`) + state.programs[programId] = { workingDirectory: relativeWorkingDir } + await setProjectState(state) + } else if (existing.workingDirectory !== relativeWorkingDir) { + throw new Error(`Conflicting programs "${programId}": existing workingDir ${existing.workingDirectory ?? ''} !== ${relativeWorkingDir ?? ''}`) + } + } + + return { + programId, + workingDirectory, + } +} + +// TODO: this should only match if the target source file is included by the config +async function tryFindTsConfig(dir: string, recurse = false) { + const config = await getFs().readFile(path.resolve(dir, 'tsconfig.json'), 'utf-8').catch(throwIfNotFileNotFoundError) + if (config) { + return { directory: dir, data: config } + } + + if (!recurse) { + return + } + + const nextDir = path.dirname(dir) + if (nextDir !== dir) { + return tryFindTsConfig(nextDir) + } +} + +async function initProjectlessProgram(id: string, workingDirectory: string) { + const state = (await getProjectState('global')) ?? { + id: 'global', + apps: {}, + programs: {}, + packages: {}, + } + + if (state && !state.programs[id]) { + getLogger().debug(`Initialized program "${id}"`) + state.programs[id] = { workingDirectory,} + await setProjectState(state) + } + + return { + programId: id, + workingDirectory, + } +} + +function getProgramIdFromDir(dir: string) { + const replaced = dir.replaceAll(path.sep, '_') + if (process.platform !== 'win32') { + return replaced + } + + if (replaced.match(/^[a-zA-Z]:/)) { + return replaced[0] + replaced.slice(2) + } + + return replaced +} + +async function findProjectlessProgram(cwd: string, target?: string) { + const targetDir = target ? path.dirname(target) : cwd + const pkg = await getPackageJson(getFs(), targetDir, false) + if (!pkg) { + const tsConfig = await tryFindTsConfig(targetDir, false) + if (tsConfig) { + const programId = getProgramIdFromDir(tsConfig.directory) + + return initProjectlessProgram(programId, tsConfig.directory) + } + + if (!target) { + const programId = getProgramIdFromDir(cwd) + + return initProjectlessProgram(programId, cwd) + // throw new Error(`No program found in cwd: ${cwd}`) + } + + if (!(await getFs().fileExists(target))) { + throw new Error(`Target file not found: ${target}`) + } + + const programId = getProgramIdFromDir(target) + + return initProjectlessProgram(programId, path.dirname(target)) + } + + const programId = getProgramIdFromDir(pkg.directory) + + return initProjectlessProgram(programId, pkg.directory) +} + +export async function findDeployment(programId: string, projectId: string, environmentName?: string): Promise { + const state = await getProjectState(projectId) + if (!state) { + throw new Error(`No project state found: ${projectId}`) + } + + const program = state.programs[programId] + if (!program?.appId) { + return !environmentName ? program?.processId : undefined + } + + const app = state.apps[program.appId] + if (!app) { + throw new Error(`Missing application: ${program.appId}`) + } + + const envName = environmentName ?? app.defaultEnvironment ?? 'local' + const environment = app.environments[envName] + if (!environment) { + return program?.processId + } + + return (environment as any).process ?? environment.deploymentId +} + +// TODO: add flag to disable auto-init +export async function resolveProgramBuildTarget(cwd: string, opt?: BuildTargetOptions): Promise { + // The target deployment is the most specific option so it's resolved first + if (opt?.deployment) { + const deployment = await getDeploymentById(opt.deployment) + const projectId = opt.project ?? deployment.projectId + // FIXME: handle `global` + const rootDirectory = await getProjectDirectory(opt.project ?? deployment.projectId) + const prog = await findProgram(cwd, rootDirectory, projectId) + + return { + projectId, + programId: prog.programId, + deploymentId: opt.deployment, + rootDirectory, + workingDirectory: prog.workingDirectory, + buildDir: path.resolve(rootDirectory, synDirName, 'build'), + environmentName: opt?.environmentName, + } + } + + const resolvedProgram = opt?.program ? path.resolve(cwd, opt.program) : undefined + const targetDir = resolvedProgram ? path.dirname(resolvedProgram) : cwd + const proj = (await findProject(targetDir)) ?? await tryInitProject(cwd) + if (!proj) { + const prog = await findProjectlessProgram(cwd, resolvedProgram) + if (!prog) { + return + } + + const projId = 'global' + const deployment = await findDeployment(prog.programId, projId, opt?.environmentName) + + return { + projectId: projId, + programId: prog.programId, + deploymentId: deployment, + rootDirectory: prog.workingDirectory, + workingDirectory: prog.workingDirectory, + buildDir: path.resolve(getUserSynapseDirectory(), 'build'), + environmentName: opt?.environmentName, + } + } + + const prog = await findProgram(targetDir, proj.rootDir, proj.id) + const deployment = await findDeployment(prog.programId, proj.id, opt?.environmentName) + + return { + projectId: proj.id, + programId: prog.programId, + deploymentId: deployment, + rootDirectory: proj.rootDir, + workingDirectory: prog.workingDirectory, + buildDir: path.resolve(getUserSynapseDirectory(), 'build'), + environmentName: opt?.environmentName, + } +} + +// * `rootDir` -> project root e.g. a `git` repo +// * `workingDir` -> how should we resolve relative paths +// * `buildDir` -> where can we put cache/build data + +export interface BuildTarget { + readonly projectId: string | 'global' + readonly programId: string // This is unique per-branch + readonly deploymentId?: string + readonly environmentName?: string + readonly rootDirectory: string // `rootDirectory` === `workingDirectory` when using a global project + readonly workingDirectory: string + readonly buildDir: string +} + +// This is captured during a heap snapshot +const shouldUseRemote = !!process.env['SYNAPSE_SHOULD_USE_REMOTE'] +const shouldCreateRemoteDeployment = false +const shouldCreateRemoteProject = shouldUseRemote + +async function createDeployment(): Promise<{ id: string; local?: boolean }> { + if (!shouldCreateRemoteDeployment || process.env['SYNAPSE_FORCE_NO_REMOTE']) { + return { id: randomUUID(), local: true } + } + + return await getProcClient().createProcess() +} + +async function _createProject(name: string, params: { url: string }): ReturnType['createProject']> { + if (!shouldCreateRemoteProject || process.env['SYNAPSE_FORCE_NO_REMOTE']) { + return { id: randomUUID(), kind: 'project', programs: {}, owner: '' } + } + + return getClient().createProject(name, params) +} + +async function getOrCreateApp(state: ProjectState, bt: BuildTarget) { + const program = state.programs[bt.programId] ?? {} + if (program.appId) { + const app = state.apps[program.appId] + if (!app) { + throw new Error(`Missing application: ${program.appId} [${bt.rootDirectory}]`) + } + + return app + } + + const app: projects.AppInfo = { id: randomUUID(), environments: {} } + program.appId = app.id + state.apps[app.id] = app + + return app +} + +async function updateProjectState(state: ProjectState) { + await setProjectState(state) + + if (shouldUseRemote) { + const ents = await getEntities() + const remote = ents.projects[state.id]?.remote + if (remote) { + await projects.client.updateProject(remote, { + apps: state.apps, + packages: state.packages, + programs: state.programs, + }) + } + } +} + +export async function getOrCreateDeployment(bt: BuildTarget = getBuildTargetOrThrow()) { + if (bt.deploymentId) { + return bt.deploymentId + } + + const state = await getProjectState(bt.projectId) + if (!state) { + throw new Error(`Missing project state: ${bt.projectId} [${bt.rootDirectory}]`) + } + + const program = state.programs[bt.programId] ?? {} + if (program?.processId && !bt.environmentName) { + return program.processId + } + + const app = await getOrCreateApp(state, bt) + + const environmentName = bt.environmentName ?? app.defaultEnvironment ?? 'local' + const environment = app.environments[environmentName] + if (environment) { + return (environment as any).process ?? environment.deploymentId + } + + const deployment = await createDeployment() + app.environments[environmentName] = { + name: environmentName, + deploymentId: deployment.id, + } + + await updateProjectState(state) + + const ents = await getEntities() + ents.deployments[deployment.id] = { + programId: bt.programId, + projectId: bt.projectId, + local: deployment.local, + } + + await setEntities(ents) + + return deployment.id +} + +interface EntitiesFile { + readonly projects: Record + readonly deployments: Record +} + +const getEntitiesFilePath = () => path.resolve(getUserSynapseDirectory(), 'entities.json') + +async function getEntities() { + const ents = await tryReadJson(getFs(), getEntitiesFilePath()) + + // Backwards compat + if (ents && (ents as any).processes && !ents.deployments) { + return { ...ents, deployments: (ents as any).processes as EntitiesFile['deployments'] } + } + + return ents ?? { projects: {}, deployments: {} } +} + +// TODO: project directories within the home dir should be made relative +async function setEntities(data: EntitiesFile) { + await getFs().writeFile(getEntitiesFilePath(), JSON.stringify(data, undefined, 4)) +} + +async function addProject(data: EntitiesFile, id: string, attr: EntitiesFile['projects'][string]) { + const entries = Object.entries(data.projects) + entries.push([id, attr]) + entries.sort((a, b) => a[1].directory.length - b[1].directory.length) + await setEntities({ ...data, projects: Object.fromEntries(entries) }) +} + +function getEntitiesSync(fs: SyncFs = getFs()) { + const ents = tryReadJsonSync(fs, getEntitiesFilePath()) + if (!ents) { + throw new Error(`No projects found`) + } + + // Backwards compat + if (ents && (ents as any).processes && !ents.deployments) { + return { ...ents, deployments: (ents as any).processes as EntitiesFile['deployments'] } + } + + return ents +} + +function getStateFilePath(projectId: string) { + return path.resolve(getUserSynapseDirectory(), 'projects', `${projectId}.json`) +} + +function migrateState(state: ProjectState): ProjectState { + if (!state.apps) { + return Object.assign(state, { apps: {} }) + } + return state +} + +async function getProjectState(projectId: string, fs = getFs()): Promise { + return tryReadJson(fs, getStateFilePath(projectId)).then(s => s ? migrateState(s) : undefined) +} + +async function setProjectState(newState: ProjectState, fs = getFs()) { + await fs.writeFile(getStateFilePath(newState.id), JSON.stringify(newState, undefined, 4)) +} + +function getProjectStateSync(projectId: string, fs: SyncFs = getFs()) { + const state = tryReadJsonSync(fs, getStateFilePath(projectId)) + if (!state) { + throw new Error(`No project state found: ${projectId}`) + } + + return migrateState(state) +} + +function findProgramByProcess(state: ProjectState, processId: string) { + for (const [k, v] of Object.entries(state.programs)) { + if (v.processId === processId) { + return k + } + + if (v.appId) { + const app = state.apps[v.appId] + if (!app) continue + + if (Object.values(app.environments).some(x => ((x as any).process ?? x.deploymentId) === processId)) { + return k + } + } + } +} + +function findDeploymentById(deploymentId: string) { + const ents = getEntitiesSync() + const deployment = ents.deployments[deploymentId] + if (!deployment) { + return + } + + const state = getProjectStateSync(deployment.projectId) + const programId = findProgramByProcess(state, deploymentId) + if (!programId) { + return + } + + return { + directory: ents.projects[deployment.projectId]?.directory, + programId, + } +} + +export function getProgramIdFromDeployment(deploymentId: string) { + const res = findDeploymentById(deploymentId) + if (!res) { + throw new Error(`No deployment found: ${deploymentId}`) + } + + return res.programId +} + +export function getRootDir(programId?: string) { + if (!programId) { + return getRootDirectory() + } + + return getRootDirectory() +} + +export function getWorkingDir(programId?: string, projectId?: string) { + if (!programId) { + return getWorkingDirectory() + } + + projectId ??= getBuildTargetOrThrow().projectId + const state = getProjectStateSync(projectId) + const prog = state.programs[programId] + if (!prog) { + // This can happen if the program attached to a process got moved + // TODO: automatically fix things for the user + throw new Error(`No program found: ${programId}`) + } + + if (projectId === 'global') { + if (!prog.workingDirectory) { + throw new Error(`Missing working directory. Corrupted program data?: ${programId}`) + } + + return prog.workingDirectory + } + + const rootDir = getProjectDirectorySync(projectId) + + return path.resolve(rootDir, prog.workingDirectory ?? '') +} + +export function getRootDirectory() { + return getBuildTargetOrThrow().rootDirectory +} + +export function getWorkingDirectory() { + return getBuildTargetOrThrow().workingDirectory +} + +export function getSynapseDir() { + const bt = getBuildTargetOrThrow() + if (bt.projectId === 'global') { + return getUserSynapseDirectory() + } + + return path.resolve(bt.rootDirectory, synDirName) +} + +export function getBuildDir(programId?: string) { + const bt = getBuildTarget() + if (!bt) { + return path.resolve(getUserSynapseDirectory(), 'build') + } + + return bt.buildDir +} + +export function getTargetDeploymentIdOrThrow(): string { + const bt = getBuildTargetOrThrow() + if (bt.deploymentId === undefined) { + throw new Error(`No deployment associated with build target: ${bt.workingDirectory}`) + } + + return bt.deploymentId +} + +interface ProjectState { + readonly id: string + readonly apps: Record + readonly programs: Record + readonly packages: Record // package name -> program id +} + +async function findProjectFromDir(dir: string) { + const ents = await getEntities() + const projects = new Map(Object.entries(ents.projects).map(([k, v]) => [v.directory, { ...v, id: k }])) + + let currentDir = dir + while (true) { + if (projects.has(currentDir)) { + return projects.get(currentDir)! + } + + const next = path.dirname(currentDir) + if (next === currentDir) { + break + } + + currentDir = next + } +} + +function getClient(): typeof projects.client { + try { + projects.client.listProjects + return projects.client + } catch { + return getBackendClient() as any + } +} + +function getProcClient(): typeof processes.client { + try { + processes.client.listProcesses + return processes.client + } catch { + return getBackendClient() as any + } +} + +async function createProject(rootDir: string, remotes?: Omit[]) { + if (!remotes) { + getLogger().warn('No git repositories found. Creating a new project without a git repo is not recommended.') + + const project = await _createProject(path.dirname(rootDir), { + url: '', + }) + + return project + } + + if (remotes.length === 0) { + // getLogger().warn('No git repositories found. Creating a new project without a git repo is not recommended.') + throw new Error(`A git repo is required to create a new project`) + } + + if (remotes.length > 1) { + // TODO: prompt user + throw new Error(`Not implemented`) + } + + const target = remotes[0] + const inferredName = target.fetchUrl.match(/\/([^\/]+)\.git$/)?.[1] ?? path.dirname(rootDir) + const project = await _createProject(inferredName, { + url: target.fetchUrl, + }) + + return project +} + +async function listRemoteProjects() { + if (!shouldCreateRemoteProject || process.env['SYNAPSE_FORCE_NO_REMOTE']) { + return [] + } + return getClient().listProjects() +} + +export async function getRemoteProjectId(projectId: string) { + const ents = await getEntities() + const proj = ents.projects[projectId] + + return proj?.remote +} + +function normalizeUrl(url: string) { + return url.replace(/\.git$/, '') +} + +async function getOrCreateRemoteProject(dir: string, remotes?: Omit[]) { + const remoteUrl = remotes?.[0].fetchUrl + const existingProjects = await listRemoteProjects() + getLogger().debug('Existing projects', existingProjects) + + const match = remoteUrl + ? existingProjects.find(p => p.gitRepository && normalizeUrl(remoteUrl) === normalizeUrl(p.gitRepository.url)) + : undefined + + if (match) { + getLogger().log(`Restoring existing project bound to remote: ${remoteUrl}`) + } else if (remoteUrl) { + getLogger().log(`Initializing new project with remote: ${remoteUrl}`) + } + + return match ?? await createProject(dir, remotes) +} + +export async function initProject(dir: string, remotes?: Omit[]) { + const remote = shouldCreateRemoteProject && !process.env['SYNAPSE_FORCE_NO_REMOTE'] + ? await getOrCreateRemoteProject(dir, remotes) : undefined + const proj = { id: randomUUID(), kind: 'project', apps: remote?.apps, programs: remote?.programs, packages: remote?.packages, owner: '' } + + const ents = await getEntities() + + if (remote?.apps) { + const appMap = new Map() + for (const [k, v] of Object.entries(remote.programs)) { + if (v.appId) { + appMap.set(v.appId, k) + } + } + for (const app of Object.values(remote.apps)) { + for (const env of Object.values(app.environments)) { + const deploymentId = (env as any).process ?? env.deploymentId + ents.deployments[deploymentId] = { + projectId: proj.id, + programId: appMap.get(app.id)!, + } + } + } + } + + await addProject(ents, proj.id, { + directory: dir, + remote: remote?.id, + }) + + const state = await getProjectState(proj.id) ?? { + id: proj.id, + apps: proj.apps ?? {}, + packages: proj.packages ?? {}, + programs: proj.programs ?? {}, + } + + await setProjectState(state) + + return { id: proj.id, rootDir: dir } +} + +async function getCurrentProjectId() { + const bt = isInContext() ? getBuildTarget() : undefined + if (bt) { + return bt.projectId + } + + const proj = await findProjectFromDir(process.cwd()) + + return proj?.id ?? 'global' +} + +export async function setPackage(pkgName: string, programId: string) { + const projectId = await getCurrentProjectId() + const state = await getProjectState(projectId) ?? { + id: projectId, + apps: {}, + programs: {}, + packages: {}, + } + + state.packages[pkgName] = programId + + await setProjectState(state) +} + +export async function listPackages(projectId?: string) { + projectId ??= await getCurrentProjectId() + const state = await getProjectState(projectId) + + return state?.packages ?? {} +} + +export async function listDeployments(id?: string) { + const projectId = id ?? await getCurrentProjectId() + + const state = await getProjectState(projectId) + if (!state) { + return {} + } + + const res: [string, string][] = [] + for (const [k, v] of Object.entries(state.programs)) { + if (!v.processId) continue + + res.push([v.processId, getWorkingDir(k)]) + } + + return Object.fromEntries(res) +} + +export async function listAllDeployments() { + const entities = await getEntities() + const res: Record = {} + for (const [k, v] of Object.entries(entities.deployments)) { + const projDir = v.projectId !== 'global' + ? entities.projects[v.projectId]?.directory + : '/' + + if (!projDir) { + continue + } + + const proj = await getProjectState(v.projectId) + if (!proj) { + continue + } + + const prog = proj.programs[v.programId] + if (!prog) { + continue + } + + const workingDirectory = path.resolve(projDir, prog.workingDirectory ?? '') + res[k] = { workingDirectory, programId: v.programId, projectId: proj.id } + } + + return res +} + +export async function deleteProject(id?: string) { + const projectId = id ?? await getCurrentProjectId() + const state = await getProjectState(projectId) + if (!state) { + return + } + + await projects.client.deleteProject(state.id) + const ent = await getEntities() + delete ent.projects[state.id] + await setEntities(ent) + + await getFs().deleteFile(getStateFilePath(projectId)) +} diff --git a/src/zig/ast.ts b/src/zig/ast.ts new file mode 100644 index 0000000..8950140 --- /dev/null +++ b/src/zig/ast.ts @@ -0,0 +1,462 @@ +import ts from 'typescript' +import { isNonNullable } from '../utils' +import { emitChunk } from '../static-solver/utils' +import { getFs } from '../execution' +import * as zig from './ast.zig' +import { getLogger } from '..' + +type SyntheticUnion> = { [P in keyof T as 'x']: { $type: P } & T[P] }['x'] + +function createSyntheticUnion>(obj: T): SyntheticUnion { + if ('$type' in obj) { + return obj as any + } + + const $type = Object.getOwnPropertyNames(obj)[0] + const inner = (obj as any)[$type] + Object.defineProperty(inner, '$type', { writable: false, enumerable: false, configurable: true, value: $type }) + + return inner +} + +interface PointerType { + readonly type: 'pointer' + readonly inner: ZigType +} + +interface IntegerType { + readonly type: 'integer' + readonly width: number // 'size' + readonly signed?: boolean +} + +interface FloatType { + readonly type: 'float' + readonly name: (typeof floatTypes)[number] +} + +interface StringType { + readonly type: 'string' + readonly length?: number + readonly mutable?: boolean + readonly nullTerminated?: boolean +} + +interface NullableType { + readonly type: 'nullable' + readonly inner: ZigType +} + +interface RecordType { + readonly type: 'record' + readonly fields: Record +} + +interface AliasType { + readonly type: 'alias' + readonly name: string +} + +type ZigType = + | StringType + | FloatType + | IntegerType + | PointerType + | NullableType + | RecordType + | AliasType + +const floatTypes = ['f16', 'f32', 'f64', 'f80', 'f128', 'c_longdouble'] + +type StringTypeNode = zig.PtrType & { child_type: { ident: zig.Identifier } } + +function isStringTypeNode(node: zig.Node): node is { ptr_type: StringTypeNode } { + const n = createSyntheticUnion(node) + if (n.$type !== 'ptr_type' || (n.size !== 'Many' && n.size !== 'Slice')) { + return false + } + + const c = createSyntheticUnion(n.child_type) + + return c.$type === 'ident' && c.name === 'u8' +} + +function convertType(name: string): ZigType | undefined { + if (floatTypes.includes(name)) { + return { + type: 'float', + name: name as FloatType['name'], + } + } + + // Zig has arbitrary bit-width types (max is 65535) + const m = name.match(/^([ui])([1-9][0-9]{0,4})$/) + if (m) { + return { + type: 'integer', + signed: m[1] === 'i', + width: Number(m[2]), + } + } + + if (name === 'usize' || name === 'isize') { + return { + type: 'integer', + signed: name === 'isize', + width: -1, + } + } + + if (name === 'FsPromise') { + return { type: 'alias', name } + } +} + +function createPromiseTypeNode(inner: ts.TypeNode) { + return ts.factory.createTypeReferenceNode('Promise', [inner]) +} + +// This type conversion assumes immutability +// Our common use-case with Zig/TypeScript interop +function toTsTypeNode(node: zig.Node): ts.TypeNode { + const n = createSyntheticUnion(node) + switch (n.$type) { + case 'ident': + if (n.name === 'anyopaque') { + return ts.factory.createTypeReferenceNode('any') + } + + if (n.name === 'bool') { + return ts.factory.createTypeReferenceNode('boolean') + } + + const converted = convertType(n.name) + if (!converted) { + return ts.factory.createTypeReferenceNode(n.name) + } + + if (converted.type === 'alias' && converted.name === 'FsPromise') { + return createPromiseTypeNode(ts.factory.createTypeReferenceNode('any')) + } + + return ts.factory.createTypeReferenceNode('number') + case 'field_access': + if (n.member === 'UTF8String') { + return ts.factory.createTypeReferenceNode('string') + } + break + case 'ptr_type': + if (isStringTypeNode(node)) { + return ts.factory.createTypeReferenceNode('string') + } + + const inner = toTsTypeNode(n.child_type) + if (n.size === 'Many' || n.size === 'Slice') { + return ts.factory.createArrayTypeNode(inner) + } + + return inner + case 'optional_type': { + const inner = toTsTypeNode(n.child_type) + + return ts.factory.createUnionTypeNode([ + inner, + ts.factory.createLiteralTypeNode(ts.factory.createNull()) + ]) + } + + } + + return ts.factory.createTypeReferenceNode('any') +} + +function fieldDeclToProperty(node: zig.FieldDecl) { + const ty = node.type_expr + if (!ty) { + return ts.factory.createPropertySignature( + undefined, + node.name, + undefined, + ts.factory.createTypeReferenceNode('any') + ) + } + + let isOptional = false + let ty2 = createSyntheticUnion(ty) + if (ty2.$type === 'optional_type') { + isOptional = true + ty2 = createSyntheticUnion(ty2.child_type) + } + + return ts.factory.createPropertySignature( + undefined, + node.name, + isOptional ? ts.factory.createToken(ts.SyntaxKind.QuestionToken) : undefined, + toTsTypeNode(ty2 as any), + ) +} + +function toTsNode(node: zig.Node, treatPubAsExport?: boolean): ts.Node | undefined { + const n = createSyntheticUnion(node) + switch (n.$type) { + case 'ident': + return ts.factory.createIdentifier(n.name) + case 'fndecl': { + const isExported = n.qualifier === 'export' || n.name === 'main' || (treatPubAsExport && n.visibility === 'pub') + if (!isExported || !n.name) { + return + } + + const mod = [ + ts.factory.createModifier(ts.SyntaxKind.ExportKeyword), + ts.factory.createModifier(ts.SyntaxKind.DeclareKeyword) + ] + + const params = n.params.map(p => { + if (!p.name) { + throw new Error(`Missing parameter name`) + } + + return ts.factory.createParameterDeclaration( + undefined, + undefined, + p.name, + undefined, + p.type_expr ? toTsTypeNode(p.type_expr) : undefined, + undefined, + ) + }) + + return ts.factory.createFunctionDeclaration( + mod, + undefined, + n.name, + undefined, + params, + n.return_type ? toTsTypeNode(n.return_type) : undefined, + undefined, + ) + } + case 'vardecl': + if (!n.initializer) { + return + } + + const c = createSyntheticUnion(n.initializer) + if (c.$type !== 'container') { + return + } + + // TODO + if (c.subtype === 'root') { + return + } + + + // TODO: we should only do this for root declarations + const mod = n.visibility === 'pub' ? [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)] : undefined + + if (c.subtype === 'enum') { + const members = c.members.map(m => { + const mm = createSyntheticUnion(m) + if (mm.$type !== 'field_decl') { + return + } + + return ts.factory.createEnumMember(mm.name, ts.factory.createStringLiteral(mm.name)) + }).filter(isNonNullable) + + return ts.factory.createEnumDeclaration(mod, n.name, members) + } + + if (c.subtype === 'struct') { + const members = c.members.map(m => { + const mm = createSyntheticUnion(m) + if (mm.$type !== 'field_decl') { + return + } + + return fieldDeclToProperty(mm) + }).filter(isNonNullable) + + return ts.factory.createInterfaceDeclaration(mod, n.name, undefined, undefined, members) + } + + if (c.subtype === 'union') { + if (!c.arg) { + throw new Error(`Missing enum tag`) + } + + const members = c.members.map(m => { + const mm = createSyntheticUnion(m) + if (mm.$type !== 'field_decl') { + return + } + + const f = fieldDeclToProperty(mm) + if (!f) { + return + } + + return ts.factory.createTypeLiteralNode([f]) + }).filter(isNonNullable) + + return ts.factory.createTypeAliasDeclaration( + mod, + n.name, + undefined, + ts.factory.createUnionTypeNode(members) + ) + } + + break + + + } +} + +type AstRoot = zig.ContainerDecl & { subtype: 'root' } + +async function getAst(sourceFile: string): Promise { + if ('parse' in (zig as any)) { + const data = await getFs().readFile(sourceFile, 'utf-8') + const rawAst = zig.parse(data) + + return JSON.parse(rawAst).container + } + + if (!('main' in (zig as any))) { + throw new Error(`Missing bindings for Zig parser`) + } + + const ast = await (zig as any).main(sourceFile) + + return ast.container +} + +function generateSourceFile(root: AstRoot, treatPubAsExport?: boolean) { + const statements = root.members.map(m => { + const n = toTsNode(m, treatPubAsExport) + if (!n || (!ts.isStatement(n) && !ts.isFunctionDeclaration(n))) { + return + } + + return n + }).filter(isNonNullable) + + const sf = ts.factory.createSourceFile(statements, ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), ts.NodeFlags.None) + + return sf +} + +interface Param { + readonly name: string + readonly type: string +} + +export interface ExportedFn { + readonly name: string + readonly params: Param[] + readonly returnType: string +} + +function toSimpleType(node: zig.Node): string { + const n = createSyntheticUnion(node) + switch (n.$type) { + case 'ident': + if (n.name === 'bool') { + return 'boolean' + } + + const converted = convertType(n.name) + if (!converted) { + throw new Error(`Unknown type: ${n.name}`) + } + + return 'number' + case 'field_access': + if (n.member === 'UTF8String') { + return 'string' + } + + break + case 'ptr_type': + if (isStringTypeNode(node)) { + return 'string' + } + + const inner = toSimpleType(n.child_type) + if (n.size === 'Many' || n.size === 'Slice') { + return 'array' // TODO + } + + return inner + case 'optional_type': { + const inner = toSimpleType(n.child_type) + + return inner + } + } + + throw new Error(`Not implemented: ${JSON.stringify(n)}`) +} + +// FIXME: this check is too simplistic +function isNativeModule(ast: AstRoot) { + return !!ast.members.map(createSyntheticUnion) + .find(m => { + if (m.$type !== 'comptime_block' || !m.block) return + const b = createSyntheticUnion(m.block) + if (b.$type !== 'block' || !b.lhs) { + return + } + + const lhs = createSyntheticUnion(b.lhs) + if (lhs.$type !== 'call_exp') { + return + } + + const exp = createSyntheticUnion(lhs.exp) + if (exp.$type === 'ident') { + return exp.name === 'registerModule' + } else if (exp.$type === 'field_access') { + // TODO: check that target is `js` module + return exp.member === 'registerModule' + } + }) +} + +export async function generateTsZigBindings(target: string) { + const ast = await getAst(target) + const isModule = isNativeModule(ast) + const sf = generateSourceFile(ast, isModule) + const sourcemapHost = { + getCurrentDirectory: () => process.cwd(), + getCanonicalFileName: (fileName: string) => fileName, + } + + const { text } = emitChunk(sourcemapHost, sf, undefined, { emitSourceMap: false, removeComments: true }) + const exportedFunctions = ast.members.map(createSyntheticUnion).filter(n => { + return n.$type === 'fndecl' + }) + .map(n => n as any as zig.FnDecl) + .filter(n => (n.qualifier === 'export' || (isModule || n.visibility === 'pub')) && n.name && n.return_type) + .map(n => ({ + name: n.name!, + params: n.params.map(p => ({ + name: p.name!, + type: isModule ? 'any' : toSimpleType(p.type_expr!), + })), + returnType: isModule ? 'any' : toSimpleType(n.return_type!), + } satisfies ExportedFn)) + + // Requires `--allowArbitraryExtensions` for tsc + const outfile = target.replace(/\.zig$/, '.d.zig.ts') + + return { + isModule, + exportedFunctions, + typeDefinition: { name: outfile, text: text }, + } +} + diff --git a/src/zig/ast.zig b/src/zig/ast.zig new file mode 100644 index 0000000..f19b910 --- /dev/null +++ b/src/zig/ast.zig @@ -0,0 +1,569 @@ +const std = @import("std"); +const zig = std.zig; +const mem = @import("./lib/mem.zig"); + +pub const Tag = enum { + ident, + block, + container, + fndecl, + vardecl, + field_decl, + ptr_type, + optional_type, + literal, + field_access, + error_union, + call_exp, + comptime_block, + parse_error_node, +}; + +pub const Node = union(Tag) { + ident: Identifier, + block: BlockNode, + container: ContainerDecl, + fndecl: FnDecl, + vardecl: VarDecl, + field_decl: FieldDecl, + ptr_type: PtrType, + optional_type: OptionalType, + literal: Literal, + field_access: FieldAccess, + error_union: ErrorUnion, + call_exp: CallExp, + comptime_block: ComptimeBlock, + parse_error_node: ParseErrorNode, +}; + +pub const LiteralSubtype = enum { + number, +}; + +pub const Literal = struct { subtype: LiteralSubtype, value: []const u8 }; + +pub const Identifier = struct { + name: []const u8, +}; + +pub const FieldAccess = struct { exp: *Node, member: []const u8 }; +pub const ErrorUnion = struct { + lhs: ?*Node, + rhs: *Node, +}; + +pub const CallExp = struct { + exp: *Node, + args: []*Node, +}; + +pub const OptionalType = struct { + child_type: *Node, +}; + +pub const PtrType = struct { + size: std.builtin.Type.Pointer.Size, + sentinel: ?*Node, + is_const: bool, + // allowzero_token: ?TokenIndex, + // const_token: ?TokenIndex, + // volatile_token: ?TokenIndex, + // main_token: TokenIndex, `*` or `**` or `[` + // align_node: Node.Index, + // addrspace_node: Node.Index, + // TODO bit_range + child_type: *Node, +}; + +pub const ContainerDecl = struct { + docs: ?[]const u8, + subtype: []const u8, + layout_token: ?[]const u8, + enum_token: ?[]const u8, + members: []*Node, + arg: ?*Node, +}; + +pub const ParamDecl = struct { + name: ?[]const u8, + is_comptime: bool, + is_variadic: bool, + is_noalias: bool, + type_expr: ?*Node, +}; + +pub const FnDecl = struct { + name: ?[]const u8, + visibility: ?[]const u8, + qualifier: ?[]const u8, // inline, export, extern + lib_name: ?[]const u8, + return_type: ?*Node, + params: []ParamDecl, + body: ?*Node, +}; + +pub const VarDecl = struct { + name: []const u8, + mutability: []const u8, + visibility: ?[]const u8, + qualifier: ?[]const u8, // inline, export, extern + lib_name: ?[]const u8, + thread_local: bool, + is_comptime: bool, + initializer: ?*Node, + // align_node: Node.Index, + // addrspace_node: Node.Index, + // section_node: Node.Index, + type_expr: ?*Node, +}; + +pub const FieldDecl = struct { + name: []const u8, + is_comptime: bool, + tuple_like: bool, + initializer: ?*Node, + align_expr: ?*Node, + type_expr: ?*Node, // Always needs to exist?? +}; + +pub const BlockNode = struct { + lhs: ?*Node, + rhs: ?*Node, +}; + +pub const ComptimeBlock = struct { + block: ?*Node, +}; + +pub const ParseErrorNode = struct { + tag: zig.Ast.Node.Tag, +}; + +const NodeConverter = struct { + gpa: std.mem.Allocator, + tree: zig.Ast, + + pub fn convert(gpa: std.mem.Allocator, tree: zig.Ast) !?*Node { + const converter = @This(){ + .gpa = gpa, + .tree = tree, + }; + + const x = tree.containerDeclRoot(); + const p = try gpa.create(Node); + + p.* = Node{ .container = ContainerDecl{ + .subtype = "root", + .layout_token = null, + .enum_token = null, + .arg = null, + .members = try converter.convert_array(x.ast.members), + .docs = try converter.maybe_get_docs(0, true), + } }; + + return p; + } + + const ConversionError = error{ + NullMember, + }; + + const Error = ConversionError || std.mem.Allocator.Error || error{NoSpaceLeft}; + + fn convert_array(self: @This(), arr: []const u32) Error![]*Node { + const new_arr = try self.gpa.alloc(*Node, arr.len); + for (0..new_arr.len) |i| { + const el = try self.convert_node(arr[i]) orelse return Error.NullMember; + new_arr[i] = el; + } + return new_arr; + } + + fn convert_node_strict(self: @This(), index: u32) Error!*Node { + return try self.convert_node(index) orelse { + if (index == 0) return Error.NullMember; + + var buf: [256]u8 = undefined; + const n: zig.Ast.Node = self.tree.nodes.get(index); + const msg = try std.fmt.bufPrint(&buf, "Failed to parse node with tag: {s}", .{@tagName(n.tag)}); + @panic(msg); + }; + } + + fn maybe_get_docs(self: @This(), token_index: u32, is_root: bool) !?[]const u8 { + const tags = self.tree.tokens.items(.tag); + var j: usize = token_index; + if (is_root) j += 1 else j -= 1; + while (j > 0 and j < self.tree.tokens.len) { + switch (tags[j]) { + .doc_comment, .container_doc_comment => { + if (is_root) j += 1 else j -= 1; + }, + else => break, + } + } + + if (j == token_index) return null; + + const range: [2]u32 = if (is_root) .{ token_index, j } else .{ j, token_index }; + const lineCount = range[1] - range[0]; + + const lines = try self.gpa.alloc([]const u8, lineCount); + var size: usize = 0; + for (0..lineCount) |i| { + const l = self.tree.tokenSlice(i + range[0]); + // lines[i] = l[3..]; + lines[i] = l; + size += lines[i].len + 1; + } + + var buf = try self.gpa.alloc(u8, size); + var pos: usize = 0; + + for (0..lineCount) |i| { + @memcpy(buf.ptr + pos, lines[i]); + pos += lines[i].len; + buf[pos] = '\n'; + pos += 1; + } + + return buf; + } + + fn create_parse_error(self: @This(), tag: zig.Ast.Node.Tag) !*Node { + const p = try self.gpa.create(Node); + p.* = Node{ .parse_error_node = ParseErrorNode{ + .tag = tag, + } }; + return p; + } + + fn convert_node(self: @This(), index: u32) !?*Node { + if (index == 0) { + return null; + } + + const n: zig.Ast.Node = self.tree.nodes.get(index); + switch (n.tag) { + .identifier => { + const p = try self.gpa.create(Node); + p.* = Node{ .ident = Identifier{ + .name = self.tree.tokenSlice(n.main_token), + } }; + + return p; + }, + .root, .container_decl_trailing, .container_decl_arg, .container_decl_arg_trailing, .container_decl_two, .container_decl_two_trailing, .tagged_union, .tagged_union_trailing, .tagged_union_enum_tag, .tagged_union_enum_tag_trailing, .tagged_union_two, .tagged_union_two_trailing, .container_decl => { + var b: [2]zig.Ast.Node.Index = undefined; + const x = self.tree.fullContainerDecl(&b, index) orelse return null; + + const members = try self.convert_array(x.ast.members); + + const p = try self.gpa.create(Node); + p.* = Node{ .container = ContainerDecl{ + .subtype = self.tree.tokenSlice(x.ast.main_token), + .layout_token = if (x.layout_token) |t| self.tree.tokenSlice(t) else null, + .enum_token = if (x.ast.enum_token) |t| self.tree.tokenSlice(t) else null, + .arg = try self.convert_node(x.ast.arg), + .members = members, + .docs = null, + } }; + + return p; + }, + .global_var_decl, .local_var_decl, .aligned_var_decl, .simple_var_decl => { + const x = self.tree.fullVarDecl(index) orelse return null; + const p = try self.gpa.create(Node); + p.* = Node{ .vardecl = VarDecl{ + .name = self.tree.tokenSlice(x.ast.mut_token + 1), + .mutability = self.tree.tokenSlice(x.ast.mut_token), + .visibility = if (x.visib_token) |t| self.tree.tokenSlice(t) else null, + .initializer = try self.convert_node(x.ast.init_node), + .qualifier = null, + .thread_local = false, + .type_expr = null, + .is_comptime = false, + .lib_name = null, + } }; + + return p; + }, + .container_field_init, .container_field_align, .container_field => { + const x = self.tree.fullContainerField(index) orelse return null; + const p = try self.gpa.create(Node); + p.* = Node{ + .field_decl = FieldDecl{ + .name = self.tree.tokenSlice(x.ast.main_token), + .initializer = try self.convert_node(x.ast.value_expr), + .type_expr = try self.convert_node(x.ast.type_expr), + .is_comptime = x.comptime_token != null, + .align_expr = null, // TODO + .tuple_like = x.ast.tuple_like, + }, + }; + + return p; + }, + .fn_proto, .fn_proto_multi, .fn_proto_one, .fn_proto_simple, .fn_decl => { + var b: [1]zig.Ast.Node.Index = undefined; + const x = self.tree.fullFnProto(&b, index) orelse return null; + const p = try self.gpa.create(Node); + var params = try self.gpa.alloc(ParamDecl, x.ast.params.len); + var iter = x.iterate(&self.tree); + while (iter.next()) |param| { + params[iter.param_i - 1] = ParamDecl{ + .name = if (param.name_token) |t| self.tree.tokenSlice(t) else null, + .type_expr = try self.convert_node(param.type_expr), + .is_comptime = false, // TODO + .is_noalias = false, // TODO + .is_variadic = param.anytype_ellipsis3 != null, + }; + } + + p.* = Node{ + .fndecl = FnDecl{ + .name = if (x.name_token) |t| self.tree.tokenSlice(t) else null, + .lib_name = null, + .body = if (n.tag == .fn_decl and n.data.rhs != 0) try self.convert_node(n.data.rhs) else null, + .visibility = if (x.visib_token) |t| self.tree.tokenSlice(t) else null, + .qualifier = if (x.extern_export_inline_token) |t| self.tree.tokenSlice(t) else null, + .return_type = try self.convert_node(x.ast.return_type), + .params = params, + }, + }; + + return p; + }, + .ptr_type_aligned, .ptr_type_sentinel, .ptr_type, .ptr_type_bit_range => { + const x = self.tree.fullPtrType(index) orelse return null; + const p = try self.gpa.create(Node); + p.* = Node{ .ptr_type = PtrType{ + .size = x.size, + .sentinel = try self.convert_node(x.ast.sentinel), + .is_const = x.const_token != null, + .child_type = try self.convert_node_strict(x.ast.child_type), + } }; + return p; + }, + .optional_type => { + const p = try self.gpa.create(Node); + p.* = Node{ .optional_type = OptionalType{ + .child_type = try self.convert_node_strict(n.data.lhs), + } }; + return p; + }, + .number_literal => { + const p = try self.gpa.create(Node); + p.* = Node{ .literal = Literal{ + .subtype = .number, + .value = self.tree.tokenSlice(n.main_token), + } }; + return p; + }, + .field_access => { + const p = try self.gpa.create(Node); + p.* = Node{ .field_access = FieldAccess{ + .exp = try self.convert_node_strict(n.data.lhs), + .member = self.tree.tokenSlice(n.data.rhs), + } }; + return p; + }, + .error_union => { + const p = try self.gpa.create(Node); + p.* = Node{ .error_union = ErrorUnion{ + .lhs = try self.convert_node(n.data.lhs), + .rhs = try self.convert_node_strict(n.data.rhs), + } }; + return p; + }, + .builtin_call_two, .builtin_call_two_comma => { + const ident = try self.gpa.create(Node); + ident.* = Node{ .ident = Identifier{ + .name = self.tree.tokenSlice(n.main_token), + } }; + + const p = try self.gpa.create(Node); + p.* = Node{ + .call_exp = CallExp{ + .exp = ident, + .args = &[_]*Node{}, // TODO + }, + }; + return p; + }, + .call_one => { + var b: [1]zig.Ast.Node.Index = undefined; + const x = self.tree.fullCall(&b, index) orelse return null; + const p = try self.gpa.create(Node); + p.* = Node{ + .call_exp = CallExp{ + .exp = try self.convert_node_strict(x.ast.fn_expr), + .args = &[_]*Node{}, + }, + }; + return p; + }, + .block, .block_semicolon, .block_two, .block_two_semicolon => { + const p = try self.gpa.create(Node); + p.* = Node{ .block = BlockNode{ + .lhs = try self.convert_node(n.data.lhs), + .rhs = try self.convert_node(n.data.rhs), + } }; + return p; + }, + .@"comptime" => { + const p = try self.gpa.create(Node); + p.* = Node{ .comptime_block = ComptimeBlock{ + .block = try self.convert_node(n.data.lhs), + } }; + return p; + }, + else => { + // return self.create_parse_error(n.tag); + }, + } + + return null; + } +}; + +// fn writeFnStdout(ctx: u8, bytes: []const u8) std.os.WriteError!usize { +// _ = ctx; +// return try std.io.getStdOut().write(bytes); +// } +// +// pub fn gen(gpa: std.mem.Allocator, file: [:0]const u8) !void { +// const source = try std.fs.createFileAbsolute(file, .{ +// .truncate = false, +// .read = true, +// }); +// const buf = try gpa.alloc(u8, 1024 * 1024 * 1024); // lol +// const size = try source.readAll(buf); +// buf[size] = 0; +// const z: [:0]u8 = buf[0..size :0]; +// const tree = try zig.Ast.parse(gpa, z, .zig); +// const root = try NodeConverter.convert(gpa, tree); + +// const writer = std.io.Writer(u8, std.os.WriteError, writeFnStdout){ .context = 0 }; +// try std.json.stringify(root, .{ .emit_null_optional_fields = false }, writer); +// } + +const WasmStreamWriter = struct { + gpa: std.mem.Allocator, + buf: []u8, + pos: usize, + + pub const Error = std.mem.Allocator.Error; + + pub fn init(gpa: std.mem.Allocator, size: usize) !@This() { + const buf = try gpa.alloc(u8, size); + + return @This(){ + .gpa = gpa, + .buf = buf, + .pos = 0, + }; + } + + fn resize(this: *@This(), new_size: usize) ![]u8 { + // std.debug.print("resize {d} -> {d}\n", .{ this.buf.len, new_size }); + // const did_resize = this.gpa.resize(this.buf, new_size); + // if (!did_resize) { + // var new_buf = try this.gpa.realloc(this.buf, new_size); + // this.buf = new_buf; + // return new_buf; // Needed on WASM ? + // } + // return this.buf; + + const new_buf = try this.gpa.realloc(this.buf, new_size); + this.buf = new_buf; + return new_buf; + } + + fn writeFn(this: *@This(), bytes: []const u8) !usize { + var buf = this.buf; + const rem = buf.len - this.pos; + if (bytes.len > rem) { + const new_size = buf.len + bytes.len; + buf = try this.resize(new_size); + } + + @memcpy(this.buf[this.pos .. this.pos + bytes.len], bytes); + //_ = mem.memcpy(buf.ptr + this.pos, bytes.ptr, bytes.len); + this.pos += bytes.len; + + return bytes.len; + } + + pub const Writer = std.io.Writer(*@This(), Error, writeFn); + + pub fn toWriter(this: *@This()) Writer { + return Writer{ .context = this }; + } + + pub fn toString(this: *@This()) ![:0]const u8 { + const size: usize = this.pos; + const buf = if (this.pos == this.buf.len) try this.resize(size + 1) else this.buf; + buf[size] = 0; + + return @ptrCast(buf); + } +}; + +// stderr: "t.zig:360:21: error: parameter of type '[:0]const u8' not allowed in function with calling convention 'C'\n" + +// 't.zig:360:21: note: slices have no guaranteed in-memory representation\n' +fn parse_ast(source: [:0]const u8) ![*:0]const u8 { + const gpa = mem.allocator; + const tree = try zig.Ast.parse(gpa, source, .zig); + const root = try NodeConverter.convert(gpa, tree); + + var writer = try WasmStreamWriter.init(gpa, 1024); + + try std.json.stringify(root, .{ .emit_null_optional_fields = false }, writer.toWriter()); + + return try writer.toString(); +} + +fn strlen(source: [*:0]const u8) usize { + var i: usize = 0; + while (source[i] != 0) i += 1; + return i; +} + +export fn parse(source: [*:0]const u8) [*:0]const u8 { + const len = strlen(source); + return parse_ast(source[0..len :0]) catch @panic("Failed to parse"); +} + +// TODO: move to `mem` +export fn alloc(size: usize) [*]u8 { + const b = mem.allocator.alloc(u8, size) catch @panic("Failed to allocate"); + + return b.ptr; +} + +// pub fn main() void {} + +// pub fn main() !void { +// const gpa = getAllocator(); +// const args = try std.process.argsAlloc(gpa); +// if (args.len == 1) { +// return error.NoFileProvided; +// } + +// const source = try std.fs.createFileAbsolute(args[1], .{ +// .truncate = false, +// .read = true, +// }); +// const buf = try gpa.alloc(u8, 1024 * 1024 * 1024); // lol +// const size = try source.readAll(buf); +// buf[size] = 0; +// const z: [:0]u8 = buf[0..size :0]; + +// const res = try parse_ast(z); +// const len = strlen(res); + +// _ = try std.io.getStdOut().write(res[0..len]); +// } + +// diff --git a/src/zig/compile.ts b/src/zig/compile.ts new file mode 100644 index 0000000..b20c252 --- /dev/null +++ b/src/zig/compile.ts @@ -0,0 +1,418 @@ +import * as path from 'node:path' +import * as builder from '../build/builder' +import { runCommand } from "../utils/process" +import { getLogger } from '..' +import { getGlobalCacheDirectory, getRootDir } from '../workspaces' +import { getFs } from '../execution' +import { ensureDir, throwIfNotFileNotFoundError } from '../utils' +import { ResolvedProgramConfig, getOutputFilename } from '../compiler/config' +import { getProgramFs } from '../artifacts' +import { ExportedFn, generateTsZigBindings } from './ast' +import { getFileHasher } from '../compiler/incremental' +import { getZigPath, registerZigProvider } from './installer' +import { downloadNodeLib } from '../cli/buildInternal' + +// FIXME: ReferenceError: Cannot access 'synDirName' before initialization +const getZigCacheDir = () => path.resolve(getGlobalCacheDirectory(), 'zig') + +function getOutFile(target: string, outDir?: string, suffix = '') { + const fileName = target.replace(/\.zig$/, suffix) + if (!outDir) { + return fileName + } + + const rootDir = getRootDir() + const rel = path.relative(rootDir, fileName) + + return path.resolve(outDir, rel) +} + +function renderStub(binaryPath: string) { + return ` +module.exports.main = function main(...args) { + const child_process = require('node:child_process') + const proc = child_process.spawn('${binaryPath}', args) + + const stdout = [] + const stderr = [] + proc.stdout?.on('data', chunk => stdout.push(chunk)) + proc.stderr?.on('data', chunk => stderr.push(chunk)) + + function getResult(chunks, parse = false) { + const buf = Buffer.concat(chunks) + const str = buf.toString('utf-8') + + return parse ? JSON.parse(str) : str + } + + return new Promise((resolve, reject) => { + proc.on('error', reject) + proc.on('close', (code, signal) => { + if (code !== 0) { + const err = Object.assign( + new Error(\`Non-zero exit code: \${code} [signal \${signal}]\`), + { code, stdout: getResult(stdout), stderr: getResult(stderr) } + ) + + reject(err) + } else { + resolve(getResult(stdout, true)) + } + }) + }) +} +` +} + +interface CompileCache { + files: Record +} + +const cacheName = `[#compile-zig]__zig-cache__.json` + +async function getCache(): Promise { + return getProgramFs().readJson(cacheName).catch(e => { + throwIfNotFileNotFoundError(e) + + return { files: {} } + }) +} + +async function setCache(data: CompileCache): Promise { + await getProgramFs().writeJson(cacheName, data) +} + +type CompileTarget = 'wasm' | 'exe' | 'dylib' | 'wasm-obj' + +function renderWasmStub(relPath: string, bindings: ExportedFn[]) { + return ` +const isSea = !!process.env.BUILDING_SEA + +let inst +function getInst() { + if (inst) { + return inst + } + + if (isSea) { + const source = require('raw-sea-asset:./${relPath}'); + const typedArray = new Uint8Array(source.buffer); + const wasmModule = new WebAssembly.Module(typedArray); + + return inst = new WebAssembly.Instance(wasmModule, { env: {} }) + } + + const fs = require('node:fs'); + const path = require('node:path'); + const source = fs.readFileSync(path.resolve(__dirname, '${relPath}')); + const typedArray = new Uint8Array(source); + + const wasmModule = new WebAssembly.Module(typedArray); + + return inst = new WebAssembly.Instance(wasmModule, { env: {} }); +} + +function allocCString(str) { + const b = Buffer.from(str) + const p = inst.exports.alloc(b.byteLength + 1) + const mem = new Uint8Array(inst.exports.memory.buffer) + for (let i = 0; i < b.byteLength; i++) { + mem[p + i] = b[i] + } + mem[p + b.byteLength] = 0 + return p +} + +function readCString(p) { + let i = p + const mem = new Uint8Array(inst.exports.memory.buffer) + while (mem[i] !== 0 && i < mem.byteLength) i++; + + const arr = mem.subarray(p, i) + return Buffer.from(arr).toString('utf-8') +} + +${bindings.map(b => { + const callParams = b.params.map(p => { + if (p.type === 'string') { + return `allocCString(${p.name})` + } + // TODO: handle signs + widths + if (p.type === 'number') { + return p.name + } + + throw new Error(`Not implemented: ${p.type}`) + }) + + const rt = b.returnType === 'string' ? `readCString(res)` : 'res' + + return ` +module.exports['${b.name}'] = function (${b.params.map(p => p.name).join(', ')}) { + const res = getInst().exports['${b.name}'](${callParams.join(', ')}) + + return ${rt} +} +` +}).join('\n')} +` +} + +// This makes things a bit more flexible (but slower) +function renderDylibStub(relPath: string, bindings: ExportedFn[]) { + return ` +const isSea = !!process.env.BUILDING_SEA + +let didInit = false +function init() { + if (didInit) { + return + } + + const path = require('node:path'); + if (isSea) { + const source = require('raw-sea-asset:./${relPath}'); + const synapseInstall = process.env.SYNAPSE_INSTALL ?? path.resolve(require('node:os').homedir(), '.synapse'); + const dest = path.resolve(synapseInstall, 'cache', 'dlls', source.hash); + const fs = require('node:fs'); + if (!fs.existsSync(dest)) { + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, new Uint8Array(source.buffer)); + } + + process.dlopen(module, dest); + didInit = true; + + return; + } + + const p = require('node:path').resolve(__dirname, '${relPath}'); + process.dlopen(module, p); + didInit = true; +} + +${bindings.map(b => { + return ` +module.exports['${b.name}'] = function (${b.params.map(p => p.name).join(', ')}) { + init(); + + return module.exports['${b.name}'](${b.params.map(p => p.name).join(', ')}) +} +` +}).join('\n')} +` +} + +registerZigProvider() + +async function runZig(file: string, outfile: string, args: string[]) { + args.push(`-femit-bin=${outfile}`) + args.push('--global-cache-dir', getZigCacheDir()) + + await ensureDir(path.dirname(outfile)) + await ensureDir(getZigCacheDir()) + + getLogger().debug(`running zig command:`, args) + + const zigPath = await getZigPath() + const out = await runCommand(zigPath, args).catch(e => { + if (!(e as any).stderr) { + throw e + } + + const errors: string[] = (e as any).stderr.split('\n') + const msg = [ + `Failed to compile "${file}"`, + ...errors.map(e => ` ${e}`) + ].join('\n') + throw new Error(msg) + }) + + if (out.trim()) { + getLogger().warn(out.trim()) + } +} + +async function buildLoadHook() { + // TODO: this file would need to be included in the app package + const targetFile = path.resolve('src', 'zig', 'win32', 'load-hook.zig') + const outfile = path.resolve('dist', 'load-hook.obj') + + await runZig(targetFile, outfile, ['build-obj', targetFile, '-target', 'x86_64-windows']) + + return outfile +} + +export async function buildWindowsShim() { + const targetFile = path.resolve('src', 'zig', 'win32', 'shim.zig') + const outfile = path.resolve('dist', 'shim.exe') + + await runZig(targetFile, outfile, ['build-exe', targetFile, '-target', 'x86_64-windows', '-O', 'ReleaseFast']) + + return outfile +} + +function getHostTarget(opt: ResolvedProgramConfig) { + const parts = opt.csc.hostTarget?.split('-') + const os = parts?.[0] + const arch = parts?.[1] + + return builder.resolveBuildTarget({ + os: os ? os as any : undefined, + arch: arch ? arch as any : undefined, + }) +} + +function toZigTarget(target: builder.QualifiedBuildTarget) { + const parts: string[] = [] + + switch (target.arch) { + case 'x64': + parts.push('x86_64') + break + + case 'aarch64': + parts.push(target.arch) + break + + default: + throw new Error(`Architecture not implemented: ${target.arch}`) + } + + switch (target.os) { + case 'darwin': + parts.push('macos') + break + + case 'linux': + case 'windows': + parts.push(target.os) + break + + default: + throw new Error(`OS not implemented: ${target.os}`) + } + + return parts.join('-') +} + +export async function compileZig(file: string, opt: ResolvedProgramConfig, target: CompileTarget = 'wasm') { + const c = await getCache() + const hash = await getFileHasher().getHash(file) + if (c.files[file]?.hash === hash) { + return + } + + const bindings = await generateTsZigBindings(file) + if (bindings.isModule) { + target = 'dylib' + } + + const exports = bindings.exportedFunctions.map(fn => `--export=${fn.name}`) + + let cmd: string + switch (target) { + case 'dylib': + cmd = 'build-lib' + break + case 'wasm-obj': + cmd = 'build-obj' + break + case 'wasm': + case 'exe': + cmd = 'build-exe' + break + } + + let extname: string | undefined + switch (target) { + case 'wasm-obj': + extname = '.o' + break + case 'dylib': + extname = '.node' + break + case 'wasm': + extname = '.wasm' + break + } + + // Can be `const` for non-wasm builds + let outfile = getOutFile(file, opt.tsc.cmd.options.outDir, extname) + + const args = [cmd, file] + const hostTarget = getHostTarget(opt) + + if (target === 'wasm' || target === 'wasm-obj') { + args.push('-target', 'wasm32-freestanding-musl') + args.push('-O', 'ReleaseFast') + } + + if (target === 'dylib') { + args.push('-dynamic', '-fallow-shlib-undefined') + args.push('-O', 'ReleaseFast') + + args.push('-target', toZigTarget(hostTarget)) + if (hostTarget.os === 'windows') { + // need to link against a `.lib` file for Windows + const libPath = await downloadNodeLib() + if (libPath) { + args.push(libPath) + } + + // TODO: using the hook requires the `delayload` MSVC feature e.g. `/delayload node.exe` + // const hookPath = await buildLoadHook() + // args.push(hookPath) + } + } + + if (target === 'wasm') { + args.push('-fno-entry', ...exports) + } + + await runZig(file, outfile, args) + + if (bindings.exportedFunctions.length > 0 || bindings.isModule) { + const wasmOutfile = outfile.replace(/\.o$/, '.wasm') + const stubText = target === 'wasm' || target === 'wasm-obj' + ? renderWasmStub(path.basename(wasmOutfile), bindings.exportedFunctions) + : target === 'exe' ? renderStub(outfile) : renderDylibStub(path.basename(outfile), bindings.exportedFunctions) + + if (target === 'wasm-obj') { + await runCommand('wasm-ld', ['-S', '--no-entry', ...exports, '-o', wasmOutfile, outfile, '--initial-memory=7340032']) + + if (!opt.csc.noInfra) { + const b = await getFs().readFile(wasmOutfile) + await getProgramFs().writeFile(`[#compile]${wasmOutfile}`, b) + } + + outfile = wasmOutfile + } + + if (target === 'dylib' || target === 'wasm') { + if (!opt.csc.noInfra) { + const b = await getFs().readFile(outfile) + await getProgramFs().writeFile(`[#compile]${outfile}`, b) + } + } + + const stubName = getOutputFilename(opt.tsc.rootDir, opt.tsc.cmd.options, file.replace(/\.zig$/, '.zig.ts')) + if (opt.csc.noInfra) { + await getFs().writeFile(stubName, stubText) + } else { + await getProgramFs().writeFile(`[#compile]${stubName}`, stubText) + await getProgramFs().writeFile(bindings.typeDefinition.name, bindings.typeDefinition.text) + } + + await getFs().writeFile(bindings.typeDefinition.name, bindings.typeDefinition.text) + } + + c.files[file] = { + ...c.files[file], + hash, + } + + await setCache(c) + + return outfile +} diff --git a/src/zig/fs-ext.ts b/src/zig/fs-ext.ts new file mode 100644 index 0000000..661b7da --- /dev/null +++ b/src/zig/fs-ext.ts @@ -0,0 +1,65 @@ +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import * as fsExt from './fs-ext.zig' + +function canUseFsExt() { + return process.release.name === 'node-synapse' +} + +export function fastCopyDir(src: string, dst: string) { + if (!canUseFsExt()) { + throw new Error(`"fastCopyDir" is not available in the current runtime`) + } + + const srcDir = path.dirname(src) + const dstDir = path.dirname(dst) + const srcBase = path.basename(src) + const dstBase = path.basename(dst) + + return fsExt.cloneDir(srcDir, srcBase, dstDir, dstBase) +} + +// Faster than `fs.rm(b, { force: true, recursive: true })` by ~50% on darwin (untested elsewhere) +export async function removeDir(dir: string) { + const files = await fs.readdir(dir, { withFileTypes: true }) + const p: Promise[] = [] + for (const f of files) { + if (!f.isDirectory()) { + p.push(fs.rm(path.resolve(dir, f.name))) + } else { + if (!canUseFsExt()) { + p.push(fs.rm(path.resolve(dir, f.name), { recursive: true, force: true })) + } else { + p.push(fsExt.removeDir(dir, f.name)) + } + } + } + + await Promise.all(p) + await fs.rmdir(dir) +} + +export async function cleanDir(dir: string, toKeep: string[]) { + const s = new Set(toKeep) + const files = await fs.readdir(dir, { withFileTypes: true }) + const p: Promise[] = [] + for (const f of files) { + if (s.has(f.name)) continue + + if (!f.isDirectory()) { + p.push(fs.rm(path.resolve(dir, f.name))) + } else { + if (!canUseFsExt()) { + p.push(fs.rm(path.resolve(dir, f.name), { recursive: true, force: true })) + } else { + p.push(fsExt.removeDir(dir, f.name)) + } + } + } + + await Promise.all(p) +} + +export async function linkBin(src: string, dst: string) { + await fsExt.symLinkBin(path.resolve(src), path.resolve(dst)) +} \ No newline at end of file diff --git a/src/zig/fs-ext.zig b/src/zig/fs-ext.zig new file mode 100644 index 0000000..625512c --- /dev/null +++ b/src/zig/fs-ext.zig @@ -0,0 +1,321 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const js = @import("./lib/js.zig"); +const fs = std.fs; + +const FsPromise = js.Promise(void, anyerror); + +const NodeErrors = error{EEXIST, ENOENT, EINVAL}; + +pub fn cloneDir(src: js.UTF8String, srcName: js.UTF8String, dst: js.UTF8String, dstName: js.UTF8String) FsPromise { + _cloneDir(src.data, srcName.data, dst.data, dstName.data) catch |e| { + return FsPromise.reject(e); + }; + + return FsPromise.resolve({}); +} + +pub fn copyDir(src: js.UTF8String, dst: js.UTF8String) FsPromise { + var srcDir = fs.openDirAbsoluteZ(src.data, .{ .iterate = true }) catch |e| return FsPromise.reject(e); + defer srcDir.close(); + + var dstDir = ensureDirAbs(dst.data) catch |e| return FsPromise.reject(e); + defer dstDir.close(); + + _copyDir(srcDir, dstDir) catch |e| return FsPromise.reject(e); + + return FsPromise.resolve({}); +} + +pub fn symLinkBin(src: js.UTF8String, dst: js.UTF8String) FsPromise { + _symLinkBin(src.data, dst.data) catch |e| return FsPromise.reject(e); + + return FsPromise.resolve({}); +} + +fn _symLinkBin(src: [:0]const u8, dst: [:0]const u8) !void { + return fs.symLinkAbsoluteZ(src, dst, .{}) catch |e| { + if (e == std.posix.SymLinkError.FileNotFound) { + return NodeErrors.ENOENT; + } + if (e != std.posix.SymLinkError.PathAlreadyExists) { + return e; + } + + try fs.deleteFileAbsoluteZ(dst); + return fs.symLinkAbsoluteZ(src, dst, .{}); + }; +} + +pub fn removeDir(parent: js.UTF8String, name: js.UTF8String) FsPromise { + _removeDir(parent.data, name.data) catch |e| return FsPromise.reject(e); + + return FsPromise.resolve({}); +} + +fn ensureDirAbs(path: [:0]const u8) !fs.Dir { + return fs.openDirAbsoluteZ(path, .{}) catch |e| { + if (e != fs.File.OpenError.FileNotFound) { + return e; + } + + fs.makeDirAbsoluteZ(path) catch |e2| { + if (e2 != std.posix.MakeDirError.PathAlreadyExists) { + return e2; + } + }; + + return fs.openDirAbsoluteZ(path, .{}); + }; +} + +fn ensureDirZ(dir: fs.Dir, path: [:0]const u8) !fs.Dir{ + return dir.openDirZ(path, .{}) catch |e| { + if (e != fs.File.OpenError.FileNotFound) { + return e; + } + + dir.makeDirZ(path) catch |e2| { + if (e2 != std.posix.MakeDirError.PathAlreadyExists) { + return e2; + } + }; + + return dir.openDirZ(path, .{}); + }; +} + +fn ensureDir(dir: fs.Dir, path: []const u8) !fs.Dir { + return dir.openDir(path, .{}) catch |e| { + if (e != fs.File.OpenError.FileNotFound) { + return e; + } + + dir.makeDir(path) catch |e2| { + if (e2 != std.posix.MakeDirError.PathAlreadyExists) { + return e2; + } + }; + + return dir.openDir(path, .{}); + }; +} + +const CopyFileRawError = error{SystemResources} || NodeErrors || std.posix.CopyFileRangeError || std.posix.SendFileError; + +const native_os = builtin.os.tag; + +// Transfer all the data between two file descriptors in the most efficient way. +// The copy starts at offset 0, the initial offsets are preserved. +// No metadata is transferred over. +fn copy_file(fd_in: std.posix.fd_t, fd_out: std.posix.fd_t, size: u64) CopyFileRawError!void { + if (comptime builtin.target.isDarwin()) { + const rc = std.posix.system.fcopyfile(fd_in, fd_out, null, std.posix.system.COPYFILE_DATA); + switch (std.posix.errno(rc)) { + .SUCCESS => return, + .INVAL => return NodeErrors.EINVAL, + .NOMEM => return error.SystemResources, + // The source file is not a directory, symbolic link, or regular file. + // Try with the fallback path before giving up. + .OPNOTSUPP => {}, + else => |err| return std.posix.unexpectedErrno(err), + } + } + + if (native_os == .linux) { + // Try copy_file_range first as that works at the FS level and is the + // most efficient method (if available). + var offset: u64 = 0; + cfr_loop: while (true) { + // The kernel checks the u64 value `offset+count` for overflow, use + // a 32 bit value so that the syscall won't return EINVAL except for + // impossibly large files (> 2^64-1 - 2^32-1). + const amt = try std.posix.copy_file_range(fd_in, offset, fd_out, offset, std.math.maxInt(u32), 0); + // Terminate as soon as we have copied size bytes or no bytes + if (amt == 0 or size == amt) break :cfr_loop; + offset += amt; + } + return; + } + + // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the + // fallback code will copy the contents chunk by chunk. + const empty_iovec = [0]std.posix.iovec_const{}; + var offset: u64 = 0; + sendfile_loop: while (true) { + const amt = try std.posix.sendfile(fd_out, fd_in, offset, 0, &empty_iovec, &empty_iovec, 0); + // Terminate as soon as we have copied size bytes or no bytes + if (amt == 0 or size == amt) break :sendfile_loop; + offset += amt; + } +} + +// https://keith.github.io/xcode-man-pages/copyfile.3.html +// https://keith.github.io/xcode-man-pages/clonefile.2.html + +const COPYFILE_RECURSIVE = 1 << 15; +const COPYFILE_CLONE = 1 << 24; +const COPYFILE_CLONE_FORCE = 1 << 25; + +extern fn copyfile(src: [*]const u8, dst: [*]const u8, state: ?*anyopaque, flags: u32) c_int; +extern fn clonefileat(src_dirfd: c_int, src: [*:0]const u8, dst_dirfd: c_int, dst: [*:0]const u8, flags: c_int) c_int; +extern fn fclonefileat(srcfd: c_int, dst_dirfd: c_int, dst: [*:0]const u8, flags: c_int) c_int; + +fn _cloneDir(src: [:0]const u8, srcName: [:0]const u8, dst: [:0]const u8, dstName: [:0]const u8) !void { + var srcDir = try fs.openDirAbsoluteZ(src.ptr, .{}); + defer srcDir.close(); + + var dstDir = try ensureDirAbs(dst); + defer dstDir.close(); + + if (!comptime builtin.target.isDarwin()) { + var nextSrc = try srcDir.openDirZ(srcName.ptr, .{ .iterate = true }); + defer nextSrc.close(); + + var nextDst = try ensureDirZ(dstDir, dstName); + defer nextDst.close(); + + return _copyDir(nextSrc, nextDst); + } + + const res = clonefileat(srcDir.fd, srcName, dstDir.fd, dstName, 0); + switch (std.posix.errno(res)) { + .SUCCESS => return, + .INVAL => return NodeErrors.EINVAL, + .NOMEM => return error.SystemResources, + .OPNOTSUPP => { + var nextSrc = try srcDir.openDirZ(srcName, .{ .iterate = true }); + defer nextSrc.close(); + + var nextDst = try ensureDirZ(dstDir, dstName); + defer nextDst.close(); + + return _copyDir(nextSrc, nextDst); + }, + .NOENT => return NodeErrors.ENOENT, + .EXIST => return NodeErrors.EEXIST, + else => |err| { + // std.debug.print("unexpected errno: {s} {d}\n", .{ @tagName(err), @intFromEnum(err) }); + + return std.posix.unexpectedErrno(err); + }, + } +} + +// std.debug.print("enoent: {s} {d} {s} {d}\n", .{ srcName, srcName.len, srcName, srcName[srcName.len] }) +fn _copyDir(srcDir: fs.Dir, dstDir: fs.Dir) !void { + var it = srcDir.iterate(); + while (try it.next()) |entry| { + switch (entry.kind) { + .directory => { + var nextSrc = try srcDir.openDir(entry.name, .{ .iterate = true }); + defer nextSrc.close(); + + var nextDst = try ensureDir(dstDir, entry.name); + defer nextDst.close(); + + try _copyDir(nextSrc, nextDst); + }, + .file => { + var in_file = try srcDir.openFile(entry.name, .{}); + defer in_file.close(); + + const st = try in_file.stat(); + const size = st.size; + const mode = st.mode; + + var out_file = try dstDir.createFile(entry.name, .{ .mode = mode }); + defer out_file.close(); + + // var atomic_file = try dstDir.atomicFile(entry.name, .{ .mode = mode }); + // defer atomic_file.deinit(); + + // try copy_file(in_file.handle, atomic_file.file.handle, size); + // try atomic_file.finish(); + + try copy_file(in_file.handle, out_file.handle, size); + }, + else => {}, + } + } +} + +// const w = std.os.windows; +// const LPCSTR = w.LPCSTR; +// const LPSECURITY_ATTRIBUTES = opaque {}; + +// // The maximum number of hard links that can be created with this function is 1023 per file. If more than 1023 links are created for a file, an error results. +// // If you pass a name longer than MAX_PATH characters to the lpFileName or lpExistingFileName parameter of the ANSI version of this function or to the Unicode version of this function without prepending "\\?\" to the path, the function returns ERROR_PATH_NOT_FOUND. + +// extern "kernel32" fn CreateHardLinkA(lpFileName: LPCSTR, lpExistingFileName: LPCSTR, lpSecurityAttributes: ?*LPSECURITY_ATTRIBUTES) callconv(w.WINAPI) bool; + +// fn _copyDirHardLink(srcDir: fs.Dir, dstDir: fs.Dir, srcDirName: [:0]const u8, dstDirName: [:0]const u8) !void { +// var it = srcDir.iterate(); + +// // This isn't correct. We're using UTF-8 encoded strings +// var buf1: [w.PATH_MAX_WIDE]u8 = undefined; +// var buf2: [w.PATH_MAX_WIDE]u8 = undefined; + +// while (try it.next()) |entry| { +// switch (entry.kind) { +// .directory => { +// var nextSrc = try srcDir.openDir(entry.name, .{ .iterate = true }); +// defer nextSrc.close(); + +// var nextDst = try ensureDir(dstDir, entry.name); +// defer nextDst.close(); + +// const nextSrcName = try std.fmt.bufPrintZ(&buf1, "{s}\\{s}", .{ srcDirName, entry.name }); +// const nextDstName = try std.fmt.bufPrintZ(&buf2, "{s}\\{s}", .{ dstDirName, entry.name }); + +// try _copyDirHardLink(nextSrc, nextDst, nextSrcName, nextDstName); +// }, +// .file => { +// const srcFile = try std.fmt.bufPrintZ(&buf1, "\\\\?\\{s}\\{s}", .{ srcDirName, entry.name }); +// const dstFile = try std.fmt.bufPrintZ(&buf2, "\\\\?\\{s}\\{s}", .{ dstDirName, entry.name }); + +// const success = CreateHardLinkA(dstFile, srcFile, null); +// if (!success) { +// const err = w.kernel32.GetLastError(); +// switch (err) { +// .SUCCESS => {}, +// else => return error.Unsupported, +// } +// } +// }, +// else => {}, +// } +// } +// } + +// $(xcrun --show-sdk-path)/usr/include/removefile.h +// +// https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/removefile.3.html +// https://keith.github.io/xcode-man-pages/removefile.3.html + +extern fn removefileat(dirfd: c_int, path: [*:0]const u8, state: ?*anyopaque, flags: c_int) c_int; + +const REMOVEFILE_RECURSIVE = 1 << 0; + +fn _removeDir(parent: [:0]const u8, path: [:0]const u8) !void { + var parentDir = try fs.openDirAbsoluteZ(parent.ptr, .{}); + defer parentDir.close(); + + if (!comptime builtin.target.isDarwin()) { + return parentDir.deleteDirZ(path.ptr); + } + + const res = removefileat(parentDir.fd, path, null, REMOVEFILE_RECURSIVE); + return switch (std.posix.errno(res)) { + .SUCCESS => {}, + .INVAL => NodeErrors.EINVAL, + .NOMEM => error.SystemResources, + .OPNOTSUPP => parentDir.deleteDirZ(path), + .NOENT => NodeErrors.ENOENT, + else => |err| std.posix.unexpectedErrno(err), + }; +} + +comptime { + js.registerModule(@This()); +} diff --git a/src/zig/gccHeaders.ts b/src/zig/gccHeaders.ts new file mode 100644 index 0000000..abfe49a --- /dev/null +++ b/src/zig/gccHeaders.ts @@ -0,0 +1,3446 @@ + +// enum encoding { +// ASCII, +// UTF8, +// BASE64, +// UCS2, +// BINARY, +// HEX, +// BUFFER, +// BASE64URL, +// LATIN1 = BINARY +// }; + +import ts from 'typescript' +import * as path from 'path' +import { getLogger } from '..' +import { getFs } from '../execution' +import { Mutable, printNodes, toSnakeCase } from '../utils' +import { runCommand } from '../utils/process' + +interface StructField { + readonly name: string + readonly type: ZigType + readonly initializer?: ZigExp +} + +interface ZigStructTypeExp { + readonly kind: 'struct-type-exp' + readonly variant?: 'packed' | 'extern' + readonly fields: StructField[] + readonly declarations: (ZigFnDecl | ZigVariableDecl)[] +} + +interface ZigEnumValue { + readonly name: string + readonly value?: ZigExp +} + +interface ZigEnumTypeExp { + readonly kind: 'enum-type-exp' + readonly variant?: 'extern' + readonly tagType?: ZigType + readonly values: ZigEnumValue[] + readonly nonExhaustive?: boolean +} + + +interface ZigVariableDecl { + readonly kind: 'variable' + readonly name: string + readonly type?: ZigType + readonly isConst?: boolean + readonly isPublic?: boolean + readonly initializer?: ZigExp +} + +// Arrays have compile-time known lengths, slices are runtime +// []T is a slice (a fat pointer, which contains a pointer of type [*]T and a length). + +// An error set type and normal type can be combined with the ! binary operator to form an error union type. +// e.g. anyerror!u64, error{InvaidChar}!void + +interface ZigSliceType { + readonly sentinel?: number + readonly length?: number | string + readonly isPointer?: string // [*]T +} + +interface ZigType { + readonly name: string + readonly isConst?: boolean + readonly isOptional?: boolean + + // TODO: flip the order + readonly pointerType?: 'c' | 'many' | 'single' // [*c]T | [*]T | *T + readonly sentinel?: number // For slices, e.g. [:0]u8 is a null-terminated string + readonly length?: number | string + readonly isArray?: boolean + readonly errorType?: ZigType + + readonly params?: { name?: string; type: ZigType }[] + readonly returnType?: ZigType + + readonly typeParams?: ZigType[] +} + +type ZigStatement = + | ZigFnDecl + | ZigVariableDecl + | ZigExpStmt + | ZigRetStmt + +interface ZigFnDecl { + readonly kind: 'fn' + // readonly isMethod?: boolean + readonly name: string + readonly isPublic?: boolean + readonly isVariadic?: boolean + readonly params: ZigParam[] + readonly returnType: ZigType + readonly body?: ZigStatement[] + readonly extern?: string | boolean +} + +interface ZigIdent { + readonly kind: 'ident' + readonly name: string +} + +interface ZigCallExp { + readonly kind: 'call-exp' + readonly exp: ZigExp + readonly args: ZigExp[] +} + +type ZigExp = + | ZigIdent + | ZigCallExp + | ZigStructTypeExp + | ZigEnumTypeExp + | ZigLiteralExp + +interface ZigExpStmt { + readonly kind: 'exp-stmt' + readonly exp: ZigExp +} + +interface ZigRetStmt { + readonly kind: 'ret-stmt' + readonly exp?: ZigExp +} + +interface ZigLiteralExp { + readonly kind: 'literal-exp' + readonly literalType: 'number' + readonly value: string +} + +function printZigStatement(stmt: ZigStatement, printer: Printer) { + switch (stmt.kind) { + case 'fn': + case 'variable': + return printDecl(stmt, printer) + case 'exp-stmt': + printExp(stmt.exp, printer) + return printer.writeLine(';') + case 'ret-stmt': + if (stmt.exp) { + printer.write('return ') + printExp(stmt.exp, printer) + printer.writeLine(';') + } else { + printer.writeLine('return;') + } + + return + } +} + +function printBody(body: ZigStatement[], printer: Printer) { + if (body.length === 0) { + printer.writeLine('{}') + return + } + + printer.writeLine('{') + printer.indent() + for (const s of body) { + printZigStatement(s, printer) + } + printer.unindent() + printer.writeLine('}') + + return +} + +function printDelimitedList(elements: T[], printFn: (el: T, printer: Printer) => void, delimeter: string, printer: Printer) { + elements.forEach((a, i) => { + printFn(a, printer) + if (i < elements.length - 1) { + printer.write(delimeter) + } + }) +} + +function printFnDecl(decl: ZigFnDecl, printer: Printer) { + // if (decl.isVariadic) { + // params.push('...') + // } + + if (decl.isPublic) { + printer.write('pub ') + } + + if (decl.extern) { + // printer.write('extern "c" ') + printer.write(`extern ${typeof decl.extern === 'string' ? `"${decl.extern}" ` : ''}`) + } + + printer.write(`fn ${decl.name}(`) + printDelimitedList(decl.params, printZigParam, ', ', printer) + printer.write(') ') + + printZigType(decl.returnType, printer) + + if (decl.body) { + printer.write(' ') + printBody(decl.body, printer) + } else { + printer.writeLine(';') + } +} + +function printEnumType(exp: ZigEnumTypeExp, printer: Printer) { + function printEnumValue(v: ZigEnumValue) { + printer.write(v.name) + if (v.value) { + printer.write(' = ') + printExp(v.value, printer) + } + + printer.writeLine(',') + } + + if (exp.variant) { + printer.write(`${exp.variant} `) + } + + printer.write('enum') + + if (exp.tagType) { + printer.write('(') + printZigType(exp.tagType, printer) + printer.write(')') + } + + printer.writeLine(' {') + printer.indent() + for (const v of exp.values) { + printEnumValue(v) + } + printer.unindent() + printer.write('}') +} + +function printExp(exp: ZigExp, printer: Printer): void { + switch (exp.kind) { + case 'literal-exp': + return printer.write(exp.value) + case 'enum-type-exp': + return printEnumType(exp, printer) + case 'struct-type-exp': + return printStructType(exp, printer) + case 'call-exp': + printExp(exp.exp, printer) + printer.write('(') + printDelimitedList(exp.args, printExp, ', ', printer) + printer.write(')') + return + case 'ident': + return printer.write(exp.name) + } + +} + +function printDecl(decl: ZigVariableDecl | ZigFnDecl, printer: Printer) { + if (decl.kind === 'variable') { + return printVarDecl(decl, printer) + } + + return printFnDecl(decl, printer) +} + +function printVarDecl(decl: ZigVariableDecl, printer: Printer) { + const prefix = `${decl.isPublic ? 'pub ' : ''}${decl.isConst ? 'const ' : 'var '}${decl.name}` + printer.write(prefix) + if (decl.type) { + printer.write(': ') + printZigType(decl.type, printer) + } + + if (decl.initializer) { + printer.write(' = ') + printExp(decl.initializer, printer) + } + + printer.writeLine(';') +} + + +function printStructType(exp: ZigStructTypeExp, printer: Printer) { + if (exp.variant) { + printer.write(`${exp.variant} `) + } + + const fields = exp.fields ?? [] + const decls = exp.declarations ?? [] + if (fields.length + decls.length === 0) { + return printer.write('struct {}') + } + + printer.writeLine('struct {') + printer.indent() + + for (const f of fields) { + printer.write(`${f.name}: `) + printZigType(f.type, printer) + + if (f.initializer) { + printer.write(' = ') + printExp(f.initializer, printer) + } + + printer.writeLine(',') + } + + if (fields.length > 0) { + printer.writeLine('') + } + + const externDecls = decls.filter(d => d.kind === 'fn' && d.extern) + for (const d of externDecls) { + printDecl(d, printer) + } + + if (externDecls.length > 0) { + printer.writeLine('') + } + + for (const d of decls) { + if (d.kind !== 'fn' || !d.extern) { + printDecl(d, printer) + } + } + + printer.unindent() + printer.write('}') +} + + + +interface ZigParam { + readonly name: string + readonly type: ZigType + readonly isComptime?: boolean +} + + +function escapeIdent(name: string) { + return `@"${name}"` +} + +function printZigType(ty: ZigType, printer: Printer) { + if (ty.returnType) { + const rt = ty.returnType + const params = ty.params! + + function printParam(p: { name?: string; type: ZigType }) { + if (p.name) { + printer.write(`${p.name}: `) + } + + printZigType(p.type, printer) + } + + printer.write('fn (') + printDelimitedList(params, printParam, ', ', printer) + printer.write(') ') + printZigType(rt, printer) + + return + } + + if (ty.isOptional) { + printer.write('?') + } + + if (ty.pointerType === 'c') { + printer.write('[*c]') + } else if (ty.pointerType === 'single') { + printer.write('*') + } else if (ty.pointerType === 'many') { + printer.write('[*]') + } + + if (ty.isConst) { + printer.write('const ') + } + + const sentinel = ty.sentinel ? `:${ty.sentinel}` : '' + const arr = ty.pointerType === 'many' ? `[*${sentinel}]` : ty.length ? `[${ty.length}${sentinel}]` : ty.isArray ? `[${sentinel}]` : '' + printer.write(arr) + printer.write(ty.name) + if (ty.typeParams) { + printer.write('(') + printDelimitedList(ty.typeParams, printZigType, ', ', printer) + printer.write(')') + } +} + +function printZigParam(param: ZigParam, printer: Printer) { + printer.write(`${param.isComptime ? 'comptime ' : ''}${param.name}: `) + printZigType(param.type, printer) +} + +function createZigGenerator() { + const statements: ZigStatement[] = [] + + function addStatement(stmt: ZigStatement) { + statements.push(stmt) + } + + function render() { + const printer = createPrinter() + for (const s of statements) { + printZigStatement(s, printer) + } + + return printer.getText() + } + + return { addStatement, render } +} + +interface Position { + offset: number + col: number + file?: string + line?: number + tokLen: number + includedFrom?: { + file: string + } +} + +interface MacroPosition { + readonly spellingLoc: Position + readonly expansionLoc: Position +} + +interface Range { + readonly begin: Position + readonly end: Position +} + +interface MacroRange { + readonly begin: MacroPosition + readonly end: MacroPosition +} + +function isMacroPos(p: Position | MacroPosition): p is MacroPosition { + return 'spellingLoc' in p && 'expansionLoc' in p +} + +function isMacroRange(r: Range | MacroRange): r is MacroRange { + return isMacroPos(r.begin) && isMacroPos(r.end) +} + +interface ClangAstNode { + readonly id: string + readonly kind: string + readonly loc?: Position + readonly range?: Range | MacroRange + readonly isImplicit?: boolean + readonly explicitlyDeleted?: boolean + readonly name?: string + readonly mangledName?: string + readonly inner?: ClangAstNode[] + readonly previousDecl?: string + // storageClass +} + +interface AstRoot extends ClangAstNode { + readonly kind: 'TranslationUnitDecl' +} + +interface Symbol { + readonly id: string + readonly fqn: string + readonly name: string + readonly decl: ClangAstNode + readonly isExtern?: boolean + readonly attributes: Attribute[] + readonly members: Record + readonly type?: CxxType + readonly complete?: boolean + + readonly visibility?: 'public' | 'private' | 'protected' +} + +interface LinkageSpecDecl extends ClangAstNode { + readonly kind: 'LinkageSpecDecl' + readonly language: string + readonly hasBraces?: boolean +} + +interface NamespaceDecl extends ClangAstNode { + readonly kind: 'NamespaceDecl' + readonly name?: string +} + +interface CxxPointerType { + readonly kind: 'pointer' + readonly inner: CxxType + readonly typeHint?: 'single' | 'multi' +} + +interface CxxRecordType { + readonly kind: 'record' + readonly variant?: 'union' + readonly members: { name: string; type: CxxType }[] +} + +export interface CxxFnType { + readonly kind: 'fn' + readonly returnType: CxxType + readonly params: { + readonly name?: string + readonly type: CxxType + }[] + readonly isVariadic?: boolean +} + +interface CxxCallExp { + readonly kind: 'call-exp' + readonly exp: CxxExp + readonly args: CxxExp[] +} + +interface CxxCommaExp { + readonly kind: 'comma-exp' + readonly args: CxxExp[] +} + +interface CxxIdentifier { + readonly kind: 'ident' + readonly text: string +} + +interface CxxTypeExp { + readonly kind: 'type-exp' + readonly type: CxxType +} + +interface CxxBinaryExp { + readonly kind: 'binary-exp' + readonly left: CxxExp + readonly right: CxxExp + readonly operator: string +} + +interface CxxParenthesizedExp { + readonly kind: 'paren-exp' + readonly expression: CxxExp +} + +interface CxxIdentWithTypes { + readonly kind: 'typed-ident' + readonly ident: CxxIdentifier + readonly typeArgs: CxxExp[] +} + +interface CxxAccessExp { + readonly kind: 'access-exp' + readonly exp: CxxExp + readonly member: string + readonly variant?: 'arrow' +} + +interface CxxElementAccessExp { + readonly kind: 'element-access-exp' + readonly exp: CxxExp + readonly arg: CxxExp +} + +interface CxxNewExp { + readonly kind: 'new-exp' + readonly exp: CxxExp + readonly args: CxxExp[] +} + +interface CxxStructInitExp { + readonly kind: 'struct-init-exp' + readonly exp?: CxxExp + readonly args: CxxExp[] +} + +interface CxxStructInitExpNamed { + readonly kind: 'struct-init-exp-named' + readonly members: Record +} + +interface CxxDeleteExp { + readonly kind: 'delete-exp' + readonly exp: CxxExp +} + +interface CxxLiteralExp { + readonly kind: 'literal-exp' + readonly value: string +} + +interface CxxTernaryExp { + readonly kind: 'ternary-exp' + readonly cond: CxxExp + readonly whenTrue: CxxExp + readonly whenFalse: CxxExp +} + +// This isn't really any expression, more like a directive +interface CxxCastExp { + readonly kind: 'cast-exp' + readonly type: CxxType + readonly exp: CxxExp +} + +interface CxxUnaryExp { + readonly kind: 'unary-exp' + readonly operator: string + readonly exp: CxxExp + readonly postfix?: boolean +} + +export type CxxExp = + | CxxIdentifier + | CxxIdentWithTypes + | CxxCallExp + | CxxAccessExp + | CxxNewExp + | CxxDeleteExp + | CxxTypeExp + | CxxStructInitExp + | CxxLiteralExp + | CxxBinaryExp + | CxxParenthesizedExp + | CxxStructInitExpNamed + | CxxElementAccessExp + | CxxTernaryExp + | CxxCastExp + | CxxUnaryExp + | CxxCommaExp + +interface CxxExpStatement { + readonly kind: 'exp-statement' + readonly exp: CxxExp +} + +interface CxxRetStatement { + readonly kind: 'ret-statement' + readonly exp?: CxxExp +} + +// Doesn't handle multiple declarators +export interface CxxVarStatement { + readonly kind: 'var-statement' + readonly name: string + readonly type: 'auto' | CxxType + readonly storage?: ('static' | 'thread_local' | 'extern' | 'register')[] + readonly initializer?: CxxExp +} + +interface CxxForStatement { + readonly kind: 'for-statement' + readonly initializer?: CxxExp + readonly condition?: CxxExp + readonly incrementor?: CxxExp + readonly statement: CxxStatement +} + +interface CxxWhileStatement { + readonly kind: 'while-statement' + readonly expression: CxxExp + readonly statement: CxxStatement +} + +interface CxxBlockStatement { + readonly kind: 'block-statement' + readonly statements: CxxStatement[] +} + +interface ControlStatement { + readonly kind: 'control-statement' + readonly variant: 'break' | 'continue' +} + +interface IfStatement { + readonly kind: 'if-statement' + readonly expression: CxxExp + readonly statement: CxxStatement + readonly else?: CxxStatement +} + + +export type CxxStatement = + | CxxTypeDefDecl + | CxxRecordDecl + | CxxExpStatement + | CxxRetStatement + | CxxVarStatement + | CxxForStatement + | CxxBlockStatement + | ControlStatement + | CxxWhileStatement + | IfStatement + +function printCxxExp(exp: CxxExp, printer: Printer) { + switch (exp.kind) { + case 'ident': + return printer.write(exp.text) + case 'new-exp': + printer.write('new ') + // falls through + case 'call-exp': + printCxxExp(exp.exp, printer) + printer.write('(') + printDelimitedList(exp.args, printCxxExp, ', ', printer) + return printer.write(')') + case 'typed-ident': + printCxxExp(exp.ident, printer) + if (exp.typeArgs) { + printer.write('<') + printDelimitedList(exp.typeArgs, printCxxExp, ', ', printer) + printer.write('>') + } + return + case 'element-access-exp': + printCxxExp(exp.exp, printer) + printer.write('[') + printCxxExp(exp.arg, printer) + return printer.write(']') + case 'access-exp': + printCxxExp(exp.exp, printer) + if (exp.variant === 'arrow') { + printer.write('->') + } else { + printer.write('.') + } + return printer.write(exp.member) + case 'delete-exp': + printer.write('delete ') + return printCxxExp(exp.exp, printer) + case 'type-exp': + return printCxxType(printer, exp.type) + case 'struct-init-exp': + if (exp.exp) { + printCxxExp(exp.exp, printer) + } + printer.write('{') + printDelimitedList(exp.args, printCxxExp, ', ', printer) + return printer.write('}') + case 'struct-init-exp-named': + printer.write('{') + printer.indent() + for (const [k, v] of Object.entries(exp.members)) { + printer.write(` .${k} = `) + printCxxExp(v, printer) + printer.writeLine(',') + } + printer.unindent() + return printer.write(' }') + case 'literal-exp': + return printer.write(exp.value) + case 'binary-exp': + printCxxExp(exp.left, printer) + printer.write(` ${exp.operator} `) + printCxxExp(exp.right, printer) + return + case 'paren-exp': + printer.write('(') + printCxxExp(exp.expression, printer) + return printer.write(')') + case 'ternary-exp': + printCxxExp(exp.cond, printer) + printer.write(' ? ') + + printCxxExp(exp.whenTrue, printer) + printer.write(' : ') + + return printCxxExp(exp.whenFalse, printer) + + case 'cast-exp': + printer.write('(') + printCxxType(printer, exp.type) + printer.write(')') + return printCxxExp(exp.exp, printer) + + case 'unary-exp': + if (!exp.postfix) { + printer.write(exp.operator) + } + printCxxExp(exp.exp, printer) + if (exp.postfix) { + printer.write(exp.operator) + } + return + case 'comma-exp': + printer.write('(') + printDelimitedList(exp.args, printCxxExp, ', ', printer) + printer.write(')') + return + } +} + +function printCxxBlock(body: CxxStatement[], printer: Printer) { + printer.writeLine('{') + printer.indent() + for (const s of body) { + printCxxStatement(s, printer) + } + printer.unindent() + printer.write('}') +} + +function printCxxVarStatement(stmt: CxxVarStatement, printer: Printer) { + if (stmt.storage) { + for (const keyword of stmt.storage) { + // Can only use this keyword on C23 or higher, otherwise it is `_Thread_local` + if (keyword === 'thread_local') { + printer.write(`_Thread_local `) + } else { + printer.write(`${keyword} `) + } + } + } + + if (stmt.type === 'auto') { + printer.write('auto') + printer.write(` ${stmt.name}`) + } else { + printCxxType(printer, stmt.type, stmt.name) + } + + if (stmt.initializer) { + printer.write(' = ') + printCxxExp(stmt.initializer, printer) + } + + return printer.writeLine(';') +} + +function printCxxStatement(stmt: CxxStatement, printer: Printer) { + switch (stmt.kind) { + case 'exp-statement': + printCxxExp(stmt.exp, printer) + return printer.writeLine(';') + case 'ret-statement': + if (stmt.exp) { + printer.write('return ') + printCxxExp(stmt.exp, printer) + } else { + printer.write('return') + } + + return printer.writeLine(';') + case 'var-statement': + return printCxxVarStatement(stmt, printer) + + case 'for-statement': + printer.write('for (') + + const exps = [stmt.initializer, stmt.condition, stmt.incrementor] + printDelimitedList(exps, exp => exp ? printCxxExp(exp, printer) : undefined, '; ', printer) + printer.write(') ') + printCxxStatement(stmt.statement, printer) + + return printer.writeLine() + case 'while-statement': + printer.write('while (') + printCxxExp(stmt.expression, printer) + printer.write(') ') + + return printCxxStatement(stmt.statement, printer) + + case 'block-statement': + return printCxxBlock(stmt.statements, printer) + + case 'control-statement': + return printer.writeLine(`${stmt.variant};`) + + case 'if-statement': + printer.write('if (') + printCxxExp(stmt.expression, printer) + printer.write(') ') + printCxxStatement(stmt.statement, printer) + if (stmt.else) { + printer.write(' else ') + printCxxStatement(stmt.else, printer) + if (stmt.else.kind !== 'if-statement') { + printer.writeLine('') + } + } else { + printer.writeLine('') + } + + return + + case 'record-decl': + case 'typedef-decl': + return printCxxDecl(stmt, printer) + } +} + +interface Scope { + readonly type: 'namespace' | 'enum' | 'record' + readonly name: string +} + +interface CxxRefType { + readonly kind: 'ref' + readonly name: string + readonly params?: CxxType[] + readonly qualifiers?: string[] + + readonly scopes?: Scope[] // The scopes in which the ref node was _found_ in + readonly symbol?: Symbol +} + +interface CxxArrayType { + readonly kind: 'array' + readonly inner: CxxType + readonly length?: string +} + +// THESE ARE TYPE NODES +export type CxxType = + | CxxArrayType + | CxxRefType + | CxxPointerType + | CxxFnType + | CxxRecordType + +interface CxxParam { + readonly name: string + readonly type: CxxType +} + +export interface CxxFunctionDecl { + readonly kind: 'fn-decl' + readonly name: string + readonly returnType: CxxType + readonly parameters: CxxParam[] + readonly isVariadic?: boolean + readonly body?: CxxStatement[] +} + +export interface CxxRecordDecl { + readonly kind: 'record-decl' + readonly name: string + readonly variant?: 'union' + readonly members: { name: string; type: CxxType }[] // Order matters! +} + +export interface CxxTypeDefDecl { + readonly kind: 'typedef-decl' + readonly name: string + readonly type: CxxType +} + + +function printCxxType(printer: Printer, type: CxxType, name?: string): void { + switch (type.kind) { + case 'ref': + printer.write(type.name) + if (type.params) { + printer.write('<') + printDelimitedList(type.params, (el, p) => printCxxType(p, el), ', ', printer) + printer.write('>') + } + if (name) { + printer.write(` ${name}`) + } + return + case 'array': + printCxxType(printer, type.inner) + if (name) { + printer.write(` ${name}`) + } else { + } + + printer.write(`[${type.length !== undefined ? `${type.length}` : ''}]`) + + return + case 'pointer': + if (type.inner.kind === 'fn') { + return printCxxType(printer, type.inner, `(*${name ?? ''})`) + } + + printCxxType(printer, type.inner) + + if (name) { + printer.write(` *${name}`) + } else { + printer.write('*') + } + + return + case 'fn': + // if (type.isVariadic) { + // params.push('...') + // } + + function printParams() { + printer.write('(') + printDelimitedList((type as CxxFnType).params, (a, p) => printCxxType(p, a.type, a.name), ', ', printer) + if ((type as CxxFnType).isVariadic) { + if ((type as CxxFnType).params.length > 0) { + printer.write(', ') + } + printer.write('...') + } + printer.write(')') + } + + if (type.returnType.kind === 'array') { + printCxxType(printer, type.returnType.inner, name) + printParams() + printer.write(`[${type.returnType.length !== undefined ? `${type.returnType.length}` : ''}]`) + return + } + + printCxxType(printer, type.returnType, name) + printParams() + + return + + case 'record': + const shouldIndent = type.members.length > 1 + printer.write(type.variant ?? 'struct') + shouldIndent ? printer.writeLine(' {') : printer.write(' { ') + shouldIndent && printer.indent() + for (const m of type.members) { + printCxxType(printer, m.type, m.name) + shouldIndent ? printer.writeLine(';') : printer.write('; ') + } + shouldIndent && printer.unindent() + printer.write('}') + + if (name) { + printer.write(` ${name}`) + } + + return + } +} + +function printCxxParam(param: CxxParam, printer: Printer) { + return printCxxType(printer, param.type, param.name) +} + +function printCxxFn(decl: CxxFunctionDecl, printer: Printer) { + printCxxType(printer, decl.returnType, decl.name) + + printer.write('(') + printDelimitedList(decl.parameters, printCxxParam, ', ', printer) + if (decl.isVariadic) { + if (decl.parameters.length > 0) { + printer.write(', ') + } + printer.write('...') + } + printer.write(')') + + if (decl.body) { + printer.write(' ') + printCxxBlock(decl.body, printer) + } + + printer.writeLine(';') + + return +} + +function printCxxRecord(decl: CxxRecordDecl, printer: Printer) { + const tagName = decl.variant ?? 'struct' + printer.writeLine(`${tagName} ${decl.name} {`) + for (const m of decl.members) { + printCxxType(printer, m.type, m.name) + printer.writeLine(';') + } + + printer.writeLine('};') + + return +} + +function printCxxDecl(decl: CxxDecl, printer: Printer) { + switch (decl.kind) { + case 'fn-decl': + return printCxxFn(decl, printer) + case 'record-decl': + return printCxxRecord(decl, printer) + case 'typedef-decl': + printer.write('typedef ') + printCxxType(printer, decl.type, decl.name) + return printer.writeLine(';') + case 'var-statement': + return printCxxVarStatement(decl, printer) + } +} + +type Printer = ReturnType +function createPrinter() { + const indentAmount = 4 + const ws = ' '.repeat(indentAmount) + const lines: string[] = [] + + let indentLevel = 0 + let currentLine = 0 + + function getText() { + return lines.join('\n') + } + + function writeLine(text: string = '') { + write(text) + currentLine += 1 + } + + function write(text: string) { + if (lines[currentLine] === undefined) { + const indent = ws.repeat(indentLevel) + lines[currentLine] = `${indent}${text}` + } else { + lines[currentLine] += text + } + } + + function indent() { + indentLevel += 1 + } + + function unindent() { + indentLevel -= 1 + } + + return { write, writeLine, getText, indent, unindent } +} + +const operands = ['+', '-', '*', '/', '%', '^', '&', '|' , '~', '!', '<', '>', '+=', '-=', '*=', '/=', '%=', '^=', '&=', '|=', '=', '==', '++', '--', '<<', '>>', '>>=', '<<=', '!=', '<=', '>=', '<=>', '&&', '||', ',', '->*', '->', '()', '[]'] + +function getOperatorOperand(decl: { name: string }) { + for (const op of operands) { + if (decl.name === `operator${op}`) { + return op + } + } +} + +type CxxDecl = + | CxxFunctionDecl + | CxxRecordDecl + | CxxTypeDefDecl + | CxxVarStatement + +export function createCxxGenerator(cOnly?: boolean) { + type LateBoundDecl = () => CxxDecl | undefined + const statements: (CxxDecl | LateBoundDecl)[] = [] + const includes = new Set() + + function addFn(decl: CxxFunctionDecl) { + statements.push(decl) + } + + function addDecl(decl: CxxDecl) { + statements.push(decl) + } + + function addLateBoundDecl(cb: LateBoundDecl) { + statements.push(cb) + } + + function addInclude(name: string) { + includes.add(name) + } + + function render() { + const printer = createPrinter() + if (includes.size > 0) { + for (const n of includes) { + printer.writeLine(`#include ${n}`) + } + printer.writeLine() + } + + if (!cOnly) { + printer.writeLine('extern "C" {') + printer.indent() + } + + for (const s of statements) { + const decl = typeof s === 'function' ? s() : s + if (decl) { + printCxxDecl(decl, printer) + } + } + + if (!cOnly) { + printer.unindent() + printer.writeLine('}') + } + + return printer.getText() + } + + return { addFn, addDecl, addInclude, addLateBoundDecl, render } +} + +function toPrimitiveType(ty: string): string | undefined { + switch(ty) { + case 'int8_t': + return 'i8' + case 'uint8_t': + case 'unsigned char': + return 'u8' + case 'int16_t': + return 'i16' + case 'uint16_t': + return 'u16' + + case 'int32_t': + return 'i32' + case 'uint32_t': + return 'u32' + case 'int64_t': + return 'i64' + case 'uint64_t': + return 'u64' + + case '__int128': + return 'i128' + case 'unsigned __int128': + return 'u128' + + case 'char': + return 'c_char' + case 'short': + return 'c_short' + case 'unsigned short': + return 'c_ushort' + case 'int': + return 'c_int' + case 'unsigned int': + return 'c_uint' + case 'long': + return 'c_long' + case 'unsigned long': + return 'c_ulong' + + case 'long long': + return 'c_longlong' + case 'unsigned long long': + return 'c_ulonglong' + case 'long double': + return 'c_longdouble' + + case '_Float16': + return 'f16' + case 'float': + return 'f32' + case 'double': + return 'f64' // f80 + case '_Float128': + return 'f128' + + + case 'intptr_t': + return 'isize' + case 'size_t': + case 'uintptr_t': + case '__darwin_size_t': + return 'usize' + + case 'ssize_t': + return 'c_long' + } +} + +function getRootType(ty: CxxRefType): CxxRefType { + if (ty.symbol?.type?.kind === 'ref') { + return getRootType(ty.symbol.type) + } + + return ty +} + + +function _generateTsBindings(symbols: Record) { + const statements: ts.Statement[] = [] + + function generateRecord(sym: Symbol) { + const d = sym.decl as RecordDeclNode + if (d.tagUsed === 'union') { + return + } + + const isSimple = d.definitionData?.isStandardLayout && d.definitionData.isTrivial + + // const externName = toSnakeCase(sym.name) + + const fields: ts.TypeElement[] = [] + // TODO: bases + + const defaultVisibility = d.tagUsed !== 'class' ? 'public' : 'private' + + // Must be done before adding fields + const decl = (sym.decl as RecordDeclNode) + if (decl.bases) { + let i = 0 + for (const b of decl.bases) { + const t = (b as any).__type + // if (t) { + // zigStruct.fields.push({ + // name: `__base${i}`, + // type: toZigType(t), + // }) + // i += 1 + // } + } + } + + for (const [k, v] of Object.entries(sym.members)) { + if (!v.type) { + continue + } + + const visibility = v.visibility ?? defaultVisibility + if (v.decl.kind === 'FieldDecl') { + if (visibility !== 'public' && (v.decl as FieldDeclNode).storageClass === 'static') { + continue + } + + const prefix = visibility !== 'public' ? '_' : '' + fields.push(ts.factory.createPropertySignature( + undefined, + `${prefix}${k}`, + undefined, + toTsType(v.type), + )) + } + + if (visibility !== 'public') { + continue + } + + if (v.decl.kind === 'CXXConstructorDecl') { + + } + + if (v.decl.kind === 'CXXMethodDecl') { + + } + } + + return fields + } + + let opaqueSymName: string | undefined + function createOpaqueType() { + if (!opaqueSymName) { + opaqueSymName = '__opaqueType' + const decl = ts.factory.createVariableDeclaration(opaqueSymName, undefined, ts.factory.createTypeReferenceNode('unique symbol')) + statements.unshift(ts.factory.createVariableStatement( + [ts.factory.createModifier(ts.SyntaxKind.DeclareKeyword)], + ts.factory.createVariableDeclarationList([decl], ts.NodeFlags.Const) + )) + } + + return ts.factory.createTypeLiteralNode([ + ts.factory.createPropertySignature( + undefined, + ts.factory.createComputedPropertyName(ts.factory.createIdentifier(opaqueSymName)), + ts.factory.createToken(ts.SyntaxKind.QuestionToken), + ts.factory.createTypeReferenceNode('unknown') + ) + ]) + } + + const renderedPrimitives = new Set() + function renderPrimitiveTypeDecl(name: string) { + if (renderedPrimitives.has(name)) { + return + } + renderedPrimitives.add(name) + if (name === 'Ptr') { + statements.unshift(ts.factory.createTypeAliasDeclaration( + undefined, + name, + [ts.factory.createTypeParameterDeclaration(undefined, 'T')], + ts.factory.createIntersectionTypeNode([ + ts.factory.createTypeReferenceNode('T'), + ts.factory.createTypeLiteralNode([ + ts.factory.createPropertySignature( + undefined, + '__ptr', + ts.factory.createToken(ts.SyntaxKind.QuestionToken), + ts.factory.createTypeReferenceNode('unknown'), + ) + ]) + ]) + )) + + return + } + + statements.unshift(ts.factory.createTypeAliasDeclaration( + undefined, + name, + undefined, + ts.factory.createIntersectionTypeNode([ + ts.factory.createTypeReferenceNode('number'), + ts.factory.createTypeLiteralNode([ + ts.factory.createPropertySignature( + undefined, + '__width', + ts.factory.createToken(ts.SyntaxKind.QuestionToken), + ts.factory.createTypeReferenceNode('unknown'), + ) + ]) + ]) + )) + } + + const renderedUnknowns = new Set() + function renderUnknownType(name: string) { + if (renderedUnknowns.has(name)) { + return + } + renderedUnknowns.add(name) + + statements.unshift(ts.factory.createTypeAliasDeclaration( + undefined, + name, + undefined, + createOpaqueType() + )) + } + + function toTsType(ty: CxxType, hint?: 'pointer-single'): ts.TypeNode { + if (ty.kind === 'ref') { + const root = getRootType(ty) + const pt = toPrimitiveType(root.name) + let name = pt ?? (root.symbol?.fqn ?? root.name) + if (name.startsWith('struct ')) { + name = name.slice('struct '.length) + } + + if (symbols[name]) { + renderSymbol(symbols[name]) + } else if (pt) { + renderPrimitiveTypeDecl(pt) + } else if (name !== 'void') { + renderUnknownType(name) + } + + return ts.factory.createTypeReferenceNode(name, root.params?.map(t => toTsType(t))) + } else if (ty.kind === 'pointer') { + if (ty.inner.kind === 'ref' && ty.inner.name === 'void') { + return ts.factory.createTypeReferenceNode('any') + } + if (ty.inner.kind === 'ref' && (ty.inner.name === 'const char' || ty.inner.name === 'char')) { + return ts.factory.createTypeReferenceNode('string') + } + renderPrimitiveTypeDecl('Ptr') + + return ts.factory.createTypeReferenceNode('Ptr', [toTsType(ty.inner)]) + } else if (ty.kind === 'array') { + // TODO: handle `sizeof...(T)` for array length + return ts.factory.createArrayTypeNode(toTsType(ty.inner)) + } else if (ty.kind === 'fn') { + return ts.factory.createFunctionTypeNode( + undefined, + ty.params.map((x, i) => ts.factory.createParameterDeclaration( + undefined, + undefined, + x.name ?? `arg_${i}`, + undefined, + toTsType(x.type), + undefined, + )), + toTsType(ty.returnType) + ) + } else if (ty.kind === 'record') { + if (ty.variant === 'union') { + return ts.factory.createUnionTypeNode( + ty.members.map(m => toTsType(m.type)) + ) + } + + return ts.factory.createTypeLiteralNode( + ty.members.map(m => ts.factory.createPropertySignature( + undefined, + m.name, + undefined, + toTsType(m.type) + )) + ) + } + + throw new Error(`Unknown type: ${(ty as any).kind}`) + } + + function numericLiteral(val: string | number) { + if ((typeof val === 'number' && val < 0) || (typeof val === 'string' && val.startsWith('-'))) { + return ts.factory.createPrefixUnaryExpression( + ts.SyntaxKind.MinusToken, + ts.factory.createNumericLiteral(typeof val === 'number' ? -val : val.slice(1)) + ) + } + return ts.factory.createNumericLiteral(val) + } + + const rendered = new Set() + function renderSymbol(sym: Symbol) { + if (rendered.has(sym)) { + return + } + + rendered.add(sym) + + if (sym.decl.kind === 'ClassTemplateDecl' && sym.decl.inner) { + + } else if (sym.decl.kind === 'EnumDecl') { + const ut = (sym.decl as & { fixedUnderylingType?: Type }).fixedUnderylingType + const tagType = ut ? toPrimitiveType(ut.desugaredQualType ?? ut.qualType) : 'c_int' + if (tagType === undefined) { + throw new Error(`Unknown enum tag type: ${ut!.desugaredQualType ?? ut!.qualType}`) + } + + const members: ts.EnumMember[] = [] + + for (const [k, v] of Object.entries(sym.members)) { + members.push(ts.factory.createEnumMember( + k.toLowerCase(), + v.decl.inner ? numericLiteral(parseEnumInitializer(v.decl)) : undefined, + )) + } + + const decl = ts.factory.createEnumDeclaration( + [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], + sym.name, + members, + ) + + statements.push(decl) + } else if (sym.decl.kind === 'FunctionDecl') { + const ty = sym.type as CxxFnType + if (!ty) { + return + } + + const lp = ty.params[ty.params.length - 1]?.type + const isVariadic = lp?.kind === 'ref' && lp.name === '...' + + const decl = ts.factory.createFunctionDeclaration( + [ + ts.factory.createModifier(ts.SyntaxKind.ExportKeyword), + ts.factory.createModifier(ts.SyntaxKind.DeclareKeyword) + ], + undefined, + sym.name, + undefined, + ty.params.map((p, i) => ts.factory.createParameterDeclaration( + undefined, + isVariadic && i === ty.params.length - 1 ? ts.factory.createToken(ts.SyntaxKind.DotDotDotToken) : undefined, + p.name ?? `arg_${i}`, + undefined, + isVariadic && i === ty.params.length - 1 + ? ts.factory.createArrayTypeNode(ts.factory.createTypeReferenceNode('any')) + : toTsType(p.type), + undefined, + )), + toTsType(ty.returnType), + undefined, + ) + + ts.addSyntheticLeadingComment(decl, ts.SyntaxKind.SingleLineCommentTrivia, ' C', true) + + statements.push(decl) + } else if (sym.decl.kind === 'CXXRecordDecl' || sym.decl.kind === 'RecordDecl') { + if (!sym.complete) { + return + } + + const v = generateRecord(sym) + if (!v) { + return + } + statements.push(ts.factory.createInterfaceDeclaration( + undefined, + sym.name, + undefined, + undefined, + v, + )) + } else if (sym.decl.kind === 'TypedefDecl' && sym.type && sym.type.kind === 'pointer') { + statements.push(ts.factory.createTypeAliasDeclaration( + undefined, + sym.name.replace(/^struct /, ''), + undefined, + toTsType(sym.type), + )) + } else if (sym.decl.kind === 'TypedefDecl' && sym.type && sym.type.kind === 'ref') { + if (sym.name === sym.type.name) { + + // Opaque type? + statements.push(ts.factory.createTypeAliasDeclaration( + undefined, + sym.name.replace(/^struct /, ''), + undefined, + createOpaqueType() + )) + } else { + statements.push(ts.factory.createTypeAliasDeclaration( + undefined, + sym.name.replace(/^struct /, ''), + undefined, + toTsType(sym.type), + )) + } + } else if (sym.decl.kind === 'TypedefDecl' && sym.type?.kind === 'record') { + + } + } + + for (const sym of Object.values(symbols)) { + if (!sym.isExtern) continue + + renderSymbol(sym) + } + + return printNodes(statements) +} + +function parseEnumInitializer(n: ClangAstNode): number | string { + const exp = n.inner?.[0] + if (!exp) { + throw new Error(`No exp to parse`) + } + + if (exp.kind === 'ImplicitCastExpr') { + return parseEnumInitializer(exp) + } + + if (exp.kind === 'ConstantExpr') { + if (exp.inner) { + + } + + const ty = (exp as any).type.qualType + if (ty === 'int' || ty === 'uint8_t' || ty === 'uint32_t' || ty === 'uint64_t' || ty === 'unsigned int') { + return (exp as any).value + } + } + + if (exp.kind === 'IntegerLiteral') { + return (exp as any).value + } + + throw new Error(`Unknown node kind: ${exp.kind} -> ${JSON.stringify(exp)}`) +} + +function generateBindings(symbols: Record) { + // We generate a header + `.cpp` file to create the bindings on the C++ side + // Then we generate a zig file to call the exported symbols + // + // The C++ side needs generated code for non-externed functions and any "non-trivial" records + // For Zig, we need to generate any structs referenced by the exported symbols + + const cxxGen = createCxxGenerator() + const zigGen = createZigGenerator() + + const symbolTable: Record = {} + for (const s of Object.values(symbols)) { + symbolTable[s.fqn] = s + } + + const root: { declarations: (ZigFnDecl | ZigVariableDecl)[] } = { declarations: [] } + const containers: Record = {} + + function findContainer(parent: { declarations: (ZigFnDecl | ZigVariableDecl)[] }, name: string) { + const decl = parent.declarations.find(x => x.name === name && x.kind === 'variable' && x.initializer?.kind === 'struct-type-exp') + if (!decl) { + return + } + + return (decl as ZigVariableDecl).initializer! as ZigStructTypeExp + } + + function getContainer(qualifier: string) { + if (containers[qualifier]) { + return containers[qualifier] + } + + let current: { declarations: (ZigFnDecl | ZigVariableDecl)[] } = root + const parts = qualifier.split('::') + + const fqn: string[] = [] + while (parts.length > 0) { + const k = parts.shift()! + fqn.push(k) + + const s = findContainer(current, k) + if (s) { + current = s + continue + } + + const ns: ZigStructTypeExp = { kind: 'struct-type-exp', variant: 'extern', fields: [], declarations: [] } + current.declarations.push({ + kind: 'variable', + name: k, + isConst: true, + isPublic: true, + initializer: ns, + }) + + current = ns + containers[fqn.join('::')] = ns + } + + return current + } + + function getParentContainer(fqn: string) { + const p = fqn.split('::').slice(0, -1).join('::') + + return getContainer(p) + } + + function toZigType(ty: CxxType, hint?: 'pointer-single'): ZigType { + let isPointer = false + let isArray = false + let arrayLength: string | undefined + let name: string | undefined + let typeParams: ZigType[] | undefined + while (true) { + if (ty.kind === 'ref') { + if ((ty as any)._useZigThis) { + name = '@This()' + hint = 'pointer-single' + break + } + + const root = getRootType(ty) + name = root.name + + if (isPointer && name === 'void') { + name = 'anyopaque' + } else { + name = toPrimitiveType(name) + if (!name) { + name = root.symbol?.fqn ?? root.name + } + } + + typeParams = root.params?.map(t => toZigType(t)) + + break + } + if (ty.kind === 'pointer') { + isPointer = true + hint ??= ty.typeHint === 'single' ? 'pointer-single' : undefined + ty = ty.inner + } else if (ty.kind === 'array') { + if (isArray) { + throw new Error(`converting multi-dimensional array type not implemented: ${JSON.stringify(ty).slice(0, 256)}`) + } + + // TODO: handle `sizeof...(T)` for array length + + isArray = true + arrayLength = ty.length + ty = ty.inner + } else if (ty.kind === 'fn') { + return { + name: '', + params: ty.params.map(x => ({ + name: x.name, + type: toZigType(x.type), + })), + returnType: toZigType(ty.returnType), + } + } + } + + return { + name: name.split('::').join('.'), + typeParams, + pointerType: isPointer ? hint === 'pointer-single' ? 'single' : 'c' : undefined, + isArray, + length: arrayLength, + } + } + + function toZigFn(decl: CxxFunctionDecl, hint?: 'method' | 'ctor'): ZigFnDecl { + return { + kind: 'fn', + name: decl.name, + params: decl.parameters.map((p, i) => ({ + name: p.name, + type: toZigType(p.type, hint === 'method' && i === 0 ? 'pointer-single' : undefined), + })), + returnType: toZigType(decl.returnType, hint === 'ctor' ? 'pointer-single' : undefined), + } + } + + // We want to unwrap any smart pointers from the interface + function convertTypes(decl: CxxFunctionDecl): CxxFunctionDecl { + function convertUniquePtr(ty: CxxRefType): CxxPointerType { + return { + kind: 'pointer', + inner: ty.params![0], + typeHint: 'single', + } + } + + function expandRef(ty: CxxType): CxxType { + if (ty.kind === 'pointer' || ty.kind === 'array') { + return { + ...ty, + inner: expandRef(ty.inner), + } + } + + if (ty.kind === 'fn') { + return { + ...ty, + returnType: expandRef(ty.returnType), + params: ty.params.map(p => ({ name: p.name, type: expandRef(p.type) })), + } + } + + if (ty.kind === 'record') { + return { + ...ty, + members: ty.members.map(x => ({ ...x, type: expandRef(x.type) })) + } + } + + return { + ...ty, + name: ty.symbol?.fqn ?? ty.name, + params: ty.params?.map(expandRef), + } + } + + + // std::shared_ptr + // std::vector + // std::string + // std::function + // std::pair + // std::initializer_list + + let returnType = expandRef(decl.returnType) + const body = [...decl.body ?? []] + const parameters = [...decl.parameters].map(p => ({ name: p.name, type: expandRef(p.type) })) + + const transforms: ((exp: CxxExp) => CxxExp)[] = [] + + for (let i = 0; i < parameters.length; i++) { + const d = parameters[i].type + if (d.kind === 'ref' && d.name === 'std::unique_ptr') { + const name = parameters[i].name + const type = convertUniquePtr(d) + parameters[i] = { name, type } + + transforms[i] = exp => { + return { + kind: 'struct-init-exp', + args: [exp], + exp: { + kind: 'typed-ident', + ident: { kind: 'ident', text: 'std::unique_ptr' }, + typeArgs: [{ kind: 'type-exp', type: type.inner }], + }, + } + } + } + } + + function transformCallLikeExp(exp: CxxExp): CxxExp { + if (exp.kind !== 'call-exp' && exp.kind !== 'new-exp') { + return exp + } + + return { + ...exp, + args: exp.args.map((a, i) => { + const fn = transforms[i] + + return fn?.(a) ?? a + }) + } + } + + if (returnType.kind === 'ref' && returnType.name === 'std::unique_ptr') { + returnType = convertUniquePtr(returnType) + for (let i = 0; i < body.length; i++) { + const s = body[i] + if (s.kind === 'ret-statement' && s.exp) { + body[i] = { + kind: 'ret-statement', + exp: { + kind: 'call-exp', + exp: { kind: 'access-exp', exp: transformCallLikeExp(s.exp), member: 'release' }, + args: [], + } + } + break + } + } + } + + if (returnType.kind === 'ref' && returnType.name === 'std::vector') { + returnType = convertUniquePtr(returnType) + for (let i = 0; i < body.length; i++) { + const s = body[i] + if (s.kind === 'ret-statement' && s.exp) { + body[i] = { + kind: 'ret-statement', + exp: { + kind: 'call-exp', + exp: { kind: 'access-exp', exp: transformCallLikeExp(s.exp), member: 'release' }, + args: [], + } + } + break + } + } + } + + return { + kind: 'fn-decl', + name: decl.name, + returnType, + parameters, + body, + isVariadic: decl.isVariadic, + } + } + + function addMethod(name: string, decl: CxxFunctionDecl, container: { declarations: ZigStructTypeExp['declarations'] }, thisType?: CxxType) { + if (thisType) { + decl.parameters.unshift({ + name: 'self', // XXX: we use `self` because `this` is not a valid C++ ident outside of a class scope + type: thisType, + }) + } + + decl = convertTypes(decl) + + cxxGen.addFn(decl) + + const externDecl: ZigFnDecl = { + ...toZigFn(decl, thisType ? undefined : name === 'init' ? 'ctor' : 'method'), + extern: true, + } + + container.declarations.push(externDecl) + + container.declarations.push({ + kind: 'fn', + name, + isPublic: true, + params: externDecl.params, + returnType: externDecl.returnType, + body: [{ + kind: 'ret-stmt', + exp: { + kind: 'call-exp', + exp: { kind: 'ident', name: externDecl.name }, + args: externDecl.params.map(p => ({ kind: 'ident', name: p.name })), + } + }], + }) + + // body: [{ + // kind: 'exp-stmt', + // exp: { + // kind: 'call-exp', + // exp: { kind: 'ident', name: externDeinit.name }, + // args: externDeinit.params.map(p => ({ kind: 'ident', name: p.name })), + // } + // }], + } + + // Instantiating classes can be done in two ways: + // 1. Heap allocation on the C++ side, returning a pointer + // 2. Allocation on the C side, pass pointer to C++ function which uses placement new + // + // For the first option, the C side must always explicitly deallocate, otherwise it's a memory leak + // For the second option, the C side needs to explicitly call non-trivial destructors before deallocation + // * Failing to do this results in undefined behavior + + function _parseEnumInitializer(n: ClangAstNode): ZigExp { + return { + kind: 'literal-exp', + literalType: 'number', + value: parseEnumInitializer(n) as any, + } + } + + function generateRecord(sym: Symbol, useZigThis?: boolean) { + const d = sym.decl as RecordDeclNode + if (d.tagUsed === 'union') { + return + } + + const isSimple = d.definitionData?.isStandardLayout && d.definitionData.isTrivial + + // const externName = toSnakeCase(sym.name) + + const zigStruct: ZigStructTypeExp = { kind: 'struct-type-exp', variant: 'extern', fields: [], declarations: [] } + + const defaultVisibility = d.tagUsed !== 'class' ? 'public' : 'private' + let didAddCtors = false + + // Must be done before adding fields + const decl = (sym.decl as RecordDeclNode) + if (decl.bases) { + let i = 0 + for (const b of decl.bases) { + const t = (b as any).__type + if (t) { + zigStruct.fields.push({ + name: `__base${i}`, + type: toZigType(t), + }) + i += 1 + } + } + } + + for (const [k, v] of Object.entries(sym.members)) { + if (!v.type) { + continue + } + + const visibility = v.visibility ?? defaultVisibility + if (v.decl.kind === 'FieldDecl') { + if (visibility !== 'public' && (v.decl as FieldDeclNode).storageClass === 'static') { + continue + } + + const prefix = visibility !== 'public' ? '_' : '' + zigStruct.fields.push({ + name: `${prefix}${k}`, + type: toZigType(v.type), + }) + } + + if (visibility !== 'public') { + continue + } + + const thisType: CxxType = { + kind: 'pointer', + inner: { + kind: 'ref', + name: sym.fqn, + _useZigThis: useZigThis, // XXX + } as CxxRefType + } + + if (v.decl.kind === 'CXXConstructorDecl') { + if (didAddCtors || v.decl.isImplicit) { + continue + // throw new Error(`Overloads not implemented`) + } + const name = `init_${sym.name}` + const t = v.type as CxxFnType + const params = t.params + const ret: CxxRetStatement = { + kind: 'ret-statement', + exp: { + kind: 'new-exp', + exp: { kind: 'ident', text: sym.fqn }, + args: params.map(p => ({ kind: 'ident', text: p.name! })), + } + } + const body = [ret] + const initDecl: CxxFunctionDecl = { + kind: 'fn-decl', + name, + body, + returnType: thisType, + parameters: params as any, + } + + addMethod('init', initDecl, zigStruct) + + // deinit + const deinitName = `deinit_${sym.name}` + const deinitDecl: CxxFunctionDecl = { + kind: 'fn-decl', + name: deinitName, + body: [{ kind: 'exp-statement', exp: { kind: 'delete-exp', exp: { kind: 'ident', text: 'self' } } }], + parameters: [], + returnType: { + kind: 'ref', + name: 'void', + }, + } + + addMethod('deinit', deinitDecl, zigStruct, thisType) + didAddCtors = true + } + + if (v.decl.kind === 'CXXMethodDecl') { + const isMaybeOperator = k.startsWith('operator') + const operand = isMaybeOperator ? getOperatorOperand(v.decl as MethodDeclNode) : undefined + if (operand) { + continue + } + + const t = v.type as CxxFnType + const name = `${sym.name}_${k}` + const returnType = t.returnType + const params = t.params as any[] + if ((v.decl as MethodDeclNode).storageClass === 'static') { + const ret: CxxRetStatement = { + kind: 'ret-statement', + exp: { + kind: 'call-exp', + exp: { kind: 'ident', text: `${sym.fqn}::${k}` }, + args: params.map(p => ({ kind: 'ident', text: p.name })), + } + } + + const fnDecl: CxxFunctionDecl = { + kind: 'fn-decl', + name, + body: [ret], + returnType, + parameters: params, + } + + addMethod(k, fnDecl, zigStruct) + } else { + const ret: CxxRetStatement = { + kind: 'ret-statement', + exp: { + kind: 'call-exp', + exp: { kind: 'access-exp', exp: { kind: 'ident', text: 'self' }, member: k, variant: 'arrow' }, + args: params.map(p => ({ kind: 'ident', text: p.name })), + } + } + + const fnDecl: CxxFunctionDecl = { + kind: 'fn-decl', + name, + returnType, + parameters: params, + body: [ret], + } + + addMethod(k, fnDecl, zigStruct, thisType) + } + } + } + + return zigStruct + } + + function generateFn(sym: Symbol, declName = sym.fqn.replace(/::/g, '_')) { + const isMaybeOperator = sym.name.startsWith('operator') + const operand = isMaybeOperator ? getOperatorOperand(sym.decl as FunctionDeclNode) : undefined + if (operand) { + return + } + + const t = sym.type as CxxFnType + const returnType = t.returnType + const params = t.params as any[] + + const ret: CxxRetStatement = { + kind: 'ret-statement', + exp: { + kind: 'call-exp', + exp: { kind: 'ident', text: sym.fqn }, + args: params.map(p => ({ kind: 'ident', text: p.name })), + } + } + + const fnDecl: CxxFunctionDecl = { + kind: 'fn-decl', + name: declName, + body: [ret], + returnType, + parameters: params, + } + + return fnDecl + } + + const rendered = new Set() + function renderSymbol(sym: Symbol) { + if (rendered.has(sym)) { + return + } + + rendered.add(sym) + + if (sym.decl.kind === 'ClassTemplateDecl' && sym.decl.inner) { + // TemplateTypeParmDecl + + const typetype: ZigType = { name: 'type' } + + const fn: ZigFnDecl = { + kind: 'fn', + body: [], + params: [], + name: sym.name, + returnType: typetype, + } + + for (const c of sym.decl.inner) { + if (c.kind === 'TemplateTypeParmDecl' && c.name) { + fn.params.push({ + name: c.name, + isComptime: true, + type: typetype, + }) + } + } + + const v = generateRecord(sym, true) + if (!v) { + throw new Error(`failed to generate record for symbol: ${sym.fqn}`) + } + fn.body!.push({ + kind: 'ret-stmt', + exp: v, + }) + + getParentContainer(sym.fqn).declarations.push(fn) + } else if (sym.decl.kind === 'EnumDecl') { + const ut = (sym.decl as & { fixedUnderylingType?: Type }).fixedUnderylingType + const tagType = ut ? toPrimitiveType(ut.desugaredQualType ?? ut.qualType) : 'c_int' + if (tagType === undefined) { + throw new Error(`Unknown enum tag type: ${ut!.desugaredQualType ?? ut!.qualType}`) + } + + const exp: ZigEnumTypeExp = { + kind: 'enum-type-exp', + variant: 'extern', + tagType: { name: tagType }, + values: [], + } + + for (const [k, v] of Object.entries(sym.members)) { + exp.values.push({ + name: k.toLowerCase(), + value: v.decl.inner ? _parseEnumInitializer(v.decl) : undefined, + }) + } + + getParentContainer(sym.fqn).declarations.push({ + kind: 'variable', + name: sym.name, + isConst: true, + initializer: exp, + }) + } else if (sym.decl.kind === 'FunctionDecl') { + const fn = generateFn(sym) + if (fn) { + addMethod(sym.name, fn, getParentContainer(sym.fqn)) + } + } else if (sym.decl.kind === 'CXXRecordDecl') { + if (!sym.complete) { + return + } + + const v = generateRecord(sym) + if (v) { + getParentContainer(sym.fqn).declarations.push({ + kind: 'variable', + name: sym.name, + isConst: true, + initializer: v, + }) + } + } + } + + for (const sym of Object.values(symbols)) { + if (!sym.isExtern) continue + + renderSymbol(sym) + } + + for (const d of root.declarations) { + zigGen.addStatement(d) + } + + return { + cxxFile: cxxGen.render(), + zigFile: zigGen.render(), + } +} + +interface Attribute { + readonly kind: string + readonly value?: string +} + +// static_cast(n) +// & Lvalue reference +// && Rvalue reference + +// #include + +function maybeParseTemplate(ty: string): CxxType | undefined { + const m = ty.match(/^([^<>\s\(\)]+)(?:<(.*)>)(?: (?[\*&]+))?(?[a-z\s]+)?$/) + if (!m) { + return + } + + const ref = { + ...parseRef(m[1]), + params: m[2] ? m[2].split(',').map(x => parseTypeStr(x)) : undefined, + } + + const isPointer = m.groups?.pointer === '*' + if (isPointer) { + return { + kind: 'pointer', + inner: ref, + } + } + + return ref +} + +function parseFnType(ty: string): CxxFnType | CxxPointerType { + const parsed = maybeParseFnType(ty) + if (!parsed) { + throw new Error(`Bad type parse: ${ty}`) + } + + return parsed +} + +function splitParams(p: string): string[] { + let i = 0, j = 0 + let parenCount = 0 + const parts: string[] = [] + for (; i < p.length; i++) { + if (p[i] === ',') { + if (parenCount === 0) { + parts.push(p.slice(j, i)) + j = i + 1 + } + } else if (p[i] === '(') { + parenCount += 1 + } else if (p[i] === ')') { + parenCount -= 1 + } + } + + if (j < i) { + parts.push(p.slice(j, i)) + } + + return parts +} + +function maybeParseFnType(ty: string): CxxFnType | CxxPointerType | undefined { + const ret = ty.match(/^(.+?) (?[\*&]+)?(?:\((?(?:\*|&)(?: [A-Za-z0-9_-]+)?)\))?\((?.*?)\)(?: (?[a-z\s]+))?(?: [&]{1,2})?$/) // XXX: NOT ROBUST + if (!ret || !ret.groups) { + return + } + + const p = ret.groups.pointer + const params = ret.groups.params + const rt = parseTypeStr(ret[1]) + + try { + const t: CxxFnType = { + kind: 'fn', + params: params === 'void' ? [] : splitParams(params).map(x => x.trim()).filter(x => !!x).map(x => ({ type: parseTypeStr(x) })), + returnType: p ? { // FIXME: need to handle double pointers + kind: 'pointer', + inner: rt, + } : rt, + } + + // Function pointer + if (ret.groups.pointer2 === '*') { + return { kind: 'pointer', inner: t } + } + + return t + } catch (e) { + console.log(ty, 'params', ret.groups.params) + throw e + } + +} + +// XXX: NOT ROBUST +function maybeParseArrayType(ty: string): CxxArrayType | undefined { + const ret = ty.match(/^(.+?)(?: (?[\*&]+))?\[(.*)\]$/) + if (!ret || !ret.groups) { + return + } + + const p = ret.groups.pointer + const lhs = parseTypeStr(ret[1]) + const inner: CxxType = p === '*' ? { kind: 'pointer', inner: lhs } : lhs + + return { + kind: 'array', + inner: inner, + length: ret[2] ? ret[2] : undefined, + } +} +// lol +// The destruction/deallocation syntax is different from what most programmers are used to, so they’ll probably screw it up. + + +// Things that should be unwrapped: +// std::unique_ptr (use `release`) +// std::shared_ptr (probably have to dyn alloc) +// std::vector -> &x[0] + +// the base class object appears first (in left-to-right order in the event of multiple inheritance), and member objects follow. +// C++ classes that contain a virtual function will have a vp somewhere in the record. It's either the +// first field or at the location of the first virtual function. + +// declarator: +// pointeropt direct-declarator +// direct-declarator: +// identifier +// ( declarator ) +// direct-declarator [ type-qualifier-listopt assignment-expressionopt ] +// direct-declarator [ static type-qualifier-listopt assignment-expression ] +// direct-declarator [ type-qualifier-list static assignment-expression ] +// direct-declarator [ type-qualifier-listopt * ] +// direct-declarator ( parameter-type-list ) +// direct-declarator ( identifier-listopt ) +// pointer: +// * type-qualifier-listopt +// * type-qualifier-listopt pointer +// type-qualifier-list: +// type-qualifier +// type-qualifier-list type-qualifier +// parameter-type-list: +// parameter-list +// parameter-list , ... +// parameter-list: +// parameter-declaration +// parameter-list , parameter-declaration +// parameter-declaration: +// declaration-specifiers declarator +// declaration-specifiers abstract-declaratoropt + +// identifier-list: +// identifier +// identifier-list , identifier + +// A declaration is [type] [ident] + +// whitespace-separated list of, in any order, +// zero or one storage-class specifiers: typedef, constexpr, auto, register, static, extern, _Thread_local +// zero or more type qualifiers: const, volatile, restrict, _Atomic +// (only when declaring functions), zero or more function specifiers: inline, _Noreturn +// zero or more alignment specifiers: _Alignas + +function parseTypeSpec(ty: string): void { + +} + +interface EmptyDeclarator { + readonly kind: 'empty' +} + +interface IdentifierDeclarator { + readonly kind: 'identifier' + readonly text: string +} + +// * declarator +interface PointerDeclarator { + readonly kind: 'pointer' + readonly qualifiers?: string[] + readonly target: Declarator +} + +// & declarator +interface ReferenceDeclarator { + readonly kind: 'reference' + readonly qualifiers?: string[] + readonly target: Declarator +} + +// declarator [] +interface ArrayDeclarator { + readonly kind: 'array' + readonly target: Declarator + readonly qualifiers?: string[] + readonly length?: '*' | string | number + readonly isStatic?: boolean +} + +// declarator () +interface FunctionDeclarator { + readonly kind: 'function' + readonly target: Declarator + readonly params: any[] + readonly qualifiers?: string[] +} + + +type Declarator = + | IdentifierDeclarator + | ReferenceDeclarator + | PointerDeclarator + | ArrayDeclarator + | FunctionDeclarator + +type NoPtrDeclarator = Exclude + +function parseDeclarator(ty: string): void { + interface ParseState { + readonly tokens: string[] + readonly refTokens: ('*' | '&')[] + readonly target?: Declarator + readonly goalType?: 'function' | 'array' + } + + const states: ParseState[] = [] + + function consumeRefTokens(refTokens: ('*' | '&')[], target: Declarator) { + while (refTokens.length > 0) { + const token = refTokens.pop()! + target = { + kind: token === '*' ? 'pointer' : 'reference', + target, + } + } + + return target + } + + function convertDecl(s: ParseState): Declarator { + if (!s.goalType) { + return { + kind: 'identifier', + text: s.tokens.join(''), + } + } + + if (s.goalType === 'function') { + if (!s.target) { + throw new Error(`Missing target`) + } + + return { + kind: 'function', + target: s.target, + params: [], + } + } + + if (s.goalType === 'array') { + if (!s.target) { + throw new Error(`Missing target`) + } + + return { + kind: 'array', + target: s.target, + + } + } + + throw new Error() + } + + function finishDeclarator(): Declarator { + const s = states.pop()! + + return consumeRefTokens(s.refTokens, convertDecl(s)) + } + + for (let i = 0; i < ty.length; i++) { + if (ty[i] === '*' || ty[i] === '&') { + const cs = states[states.length - 1] + cs.refTokens.push(ty[i] as '*' | '&') + } else if (ty[i] === '(') { + // Parse fn declarator + const cs = states[states.length - 1] + + if (cs.refTokens.length === 0) { + const d = finishDeclarator() + states.push({ + tokens: [], + refTokens: [], + target: d, + goalType: 'function', + }) + } else { + // Parse inner declarator + states.push({ + tokens: [], + refTokens: [], + }) + } + } else if (ty[i] === ')' || ty[i] === ']') { + const d = finishDeclarator() + const cs = states[states.length - 1] + if (!cs.target) { + (cs as any).target = d + } + } else if (ty[i] === '[') { + const cs = states[states.length - 1] + + if (cs.refTokens.length === 0) { + const d = finishDeclarator() + states.push({ + tokens: [], + refTokens: [], + target: d, + goalType: 'array', + }) + } else { + (cs as any).goalType === 'array' + } + } + } +} + +function parseRef(ref: string): CxxRefType { + const parts = ref.split(' ') + + return { + kind: 'ref', + name: parts.pop()!, + qualifiers: parts, + } +} + +function parseTypeStr(ty: string): CxxType { + const t2 = maybeParseArrayType(ty) + if (t2) { + return t2 + } + + const t1 = maybeParseFnType(ty) + if (t1) { + return t1 + } + + const t = maybeParseTemplate(ty) + if (t) { + return t + } + + const r = ty.match(/^(?:struct )?([^&\*]+)(?: (?\*const))?(?: (?[\*&]+))?(?: (?[a-z]+))?$/) + if (!r) { + if (ty.endsWith('*restrict')) { + return parseTypeStr(ty.slice(0, -'restrict'.length)) + } + return { kind: 'ref', name: 'unknown' } + throw new Error(`Bad parse: ${ty}`) + } + + const ptrType = r.groups?.pointer + if (ptrType === '*' || r.groups?.constPointer) { + return { kind: 'pointer', inner: parseRef(r[1]) } + } else if (ptrType === '**') { + return { kind: 'pointer', inner: { kind: 'pointer', inner: parseRef(r[1]) } } + } + + return parseRef(r[1]) +} + +// Converts the AST into a symbolic program +function resolveAst(ast: AstRoot, targetFiles: string[]) { + let currentFile: string | undefined + let isExtern = false + let prevNode: ClangAstNode | undefined = undefined + + const scopes: Scope[] = [] + const visibility: ('public' | 'private' | 'protected' | undefined)[] = [] + const symbols: Record = {} + + const namespaces: Record> = {} + + function getNamespace(depth?: number) { + const fqn = scopes.map(s => s.name).slice(0, depth).join('::') + + return namespaces[fqn] ??= {} + } + + function getGlobalNamespace() { + return namespaces[''] ??= {} + } + + if (!ast.inner) { + throw new Error(`AST has no children nodes`) + } + + function visitChildren(n: ClangAstNode) { + if (n.inner) { + for (const c of n.inner) { + visit(c) + } + } + } + + function getSource() { + if (!currentFile) { + throw new Error(`No source found`) + } + return getFs().readFileSync(currentFile, 'utf-8') + } + + function parseAttr(n: ClangAstNode) { + if (!n.range) { + throw new Error(`Missing range in attr node: ${n}`) + } + + const start = isMacroRange(n.range) ? n.range.begin.spellingLoc.offset : n.range.begin.offset + const end = isMacroRange(n.range) ? n.range.end.expansionLoc.offset : n.range.end.offset + const text = getSource().slice(start, end) + const m = text.match(/(.*)(?:\((.*)\))?/) + if (!m) { + throw new Error(`Bad attr parse: ${text}`) + } + + return { + kind: m[1], + value: m[2], + } + } + + const targets = new Set(targetFiles) + function isTargetSym() { + if (!currentFile) { + return false + } + + return targets.has(currentFile) + } + + const currentAttrs: Attribute[] = [] + + visitChildren(ast) + + function createSymbol(n: ClangAstNode, fqn = [...scopes.map(s => s.name), n.name].join('::')) { + const id = n.previousDecl ?? n.id + if (symbols[id]) { + return symbols[id] + } + + if (!n.name) { + throw new Error(`Expected node to have a name: ${JSON.stringify(n)}`) + } + + const sym: Symbol = { + id: id, + fqn, + name: n.name, + decl: n, + isExtern: isTargetSym(), + attributes: [], + members: {}, + visibility: visibility[visibility.length - 1], + } + + symbols[id] = sym + + return sym + } + + function createGlobalSymbol(n: ClangAstNode) { + const sym = createSymbol(n) + getNamespace()[sym.name] = sym + + return sym + } + + function getSymbol(name: string) { + return getNamespace()[name] + } + + function getContainerSymbol(name: string) { + return getNamespace(-1)[name] + } + + function addToParent(sym: Symbol) { + if (scopes.length === 0) { + return + } + + const p = getContainerSymbol(scopes[scopes.length - 1].name) + if (p) { + p.members[sym.name] = sym + } + } + + function addNamespaceToType(ty: CxxType): void { + if (ty.kind === 'ref') { + if (ty.scopes !== undefined) { + return + } + + ;(ty as Mutable).scopes = [...scopes] + } else if (ty.kind === 'pointer' || ty.kind === 'array') { + return addNamespaceToType(ty.inner) + } else if (ty.kind === 'fn') { + addNamespaceToType(ty.returnType) + for (const p of ty.params) { + addNamespaceToType(p.type) + } + } + } + + // Should only be called after parsing everything + function addSymbolToRef(ref: CxxRefType, scopes = ref.scopes): void { + if (ref.symbol !== undefined || scopes === undefined) { + return + } + + if (ref.params) { + for (const p of ref.params) { + if (p.kind === 'ref') { + addSymbolToRef(p, scopes) + } + } + } + + // Only check in the global scope + if (ref.name.startsWith('::')) { + const sym = namespaces['']?.[ref.name.slice(2)] + ;(ref as Mutable).symbol = sym + + return + } + + const parts = ref.name.split('::') + const symName = parts.pop()! + + const scope = scopes.map(x => x.name) + for (let i = scope.length; i >= 0; i--) { + const ns = [...scope.slice(0, i), ...parts].join('::') + const sym = namespaces[ns]?.[symName] + if (sym) { + ;(ref as Mutable).symbol = sym + break + } + } + } + + function addSymbols(ty: CxxType): void { + if (ty.kind === 'ref') { + addSymbolToRef(ty) + } else if (ty.kind === 'array' || ty.kind === 'pointer') { + addSymbols(ty.inner) + } else if (ty.kind === 'fn') { + addSymbols(ty.returnType) + for (const p of ty.params) { + addSymbols(p.type) + } + } + } + + function visit(n: ClangAstNode) { + if (!n.loc) { + return + } + + if (n.loc.file) { + currentFile = n.loc.file + } else if (n.range && isMacroPos(n.range.begin) && n.range.begin.expansionLoc.file) { + currentFile = n.range.begin.expansionLoc.file + } + + if (n.kind === 'NamespaceDecl') { + const d = n as NamespaceDecl + const name = d.name ?? `__${Object.keys(namespaces).length}` // XXX: MAKE THIS GOOD + const ns = createGlobalSymbol({ ...d, name }) + addToParent(ns) + scopes.push({ type: 'namespace', name }) + visitChildren(n) + scopes.pop() + + return + } + + if (n.kind === 'EnumDecl') { + // Enums effectively create a namespace + const s = n.name ? createGlobalSymbol(n) : createGlobalSymbol({ ...n, name: `__anon_${n.id}` }) + addToParent(s) + scopes.push({ type: 'enum', name: s.name }) + visitChildren(n) + scopes.pop() + + return + } + + if (n.kind === 'FriendDecl') { + return + } + + if (n.kind === 'ClassTemplateDecl' && n.name) { + const record = n.inner?.find(c => c.kind === 'CXXRecordDecl') + if (!record?.inner || !(record as RecordDeclNode).completeDefinition) { + return + } + + ;(n as any).bases = (record as any).bases + ;(n as any).tagUsed = (record as any).tagUsed + ;(n as any).definitionData = (record as any).definitionData + const ns = createGlobalSymbol(n) + + addToParent(ns) + visibility.push(undefined) + scopes.push({ type: 'record', name: ns.name }) + visitChildren(record) + scopes.pop() + visibility.pop() + + ;(ns as Mutable).complete = true + + // TemplateTypeParmDecl + return + } + + // FunctionTemplateDecl + + if (n.kind === 'EnumConstantDecl' && n.name) { + addToParent(createSymbol(n)) + return + } + + if (n.kind === 'LinkageSpecDecl' && (n as LinkageSpecDecl).language === 'C') { + isExtern = true + visitChildren(n) + isExtern = false + + return + } + + // [[nodiscard]] + // [[no_unique_address]] <-- can change record layout + // [[clang::trivial_abi]] + // __attribute__((trivial_abi)) + + // Attributes affect the next decl we see + if (n.kind === 'VisibilityAttr') { + const attr = parseAttr(n) + currentAttrs.push(attr) + return + } + + function _addParamNames(fnType: CxxFnType, p: ClangAstNode) { + if (!p.inner) { + return + } + + let i = 0 + for (const c of p.inner) { + const t = fnType.params[i]?.type ?? { kind: 'ref', name: 'unknown' } + if (c.kind === 'ParmVarDecl') { + fnType.params[i] = { + name: c.name ?? `arg_${i}`, + type: t, + } + i += 1 + } + } + } + + function addParamNames(fnType: CxxType, p: ClangAstNode): void { + if (!p.inner) { + return + } + + if (fnType.kind === 'fn') { + return _addParamNames(fnType, p) + } + + if (fnType.kind === 'pointer') { + return addParamNames(fnType.inner, p) + } + + throw new Error(`Unknown type kind: ${fnType.kind}`) + } + + if (n.kind === 'CXXMethodDecl' && n.name) { + if ((n as any).isImplicit || n.explicitlyDeleted) { + return + } + + const s = getContainerSymbol(scopes[scopes.length - 1].name) + + if (s && (s.decl.kind === 'CXXRecordDecl' || s.decl.kind === 'ClassTemplateDecl')) { + const ms = s.members[n.name] = createSymbol(n) + if ((n as any).type) { + const ty = (n as any).type as Type + const fnType = parseFnType(ty.desugaredQualType ?? ty.qualType) + ;(ms as Mutable).type = fnType + + addParamNames(fnType, n) + addNamespaceToType(fnType) + } + } + + return + } + + if (n.kind === 'CXXConstructorDecl') { + if ((n as any).isImplicit || n.explicitlyDeleted) { + return + } + + const s = getContainerSymbol(scopes[scopes.length - 1].name) + const symName = '__constructor__' + if (s && (s.decl.kind === 'CXXRecordDecl' || s.decl.kind === 'ClassTemplateDecl')) { + const ms = s.members[symName] = createSymbol(n) + if ((n as any).type) { + const ty = (n as any).type as Type + const fnType = parseFnType(ty.desugaredQualType ?? ty.qualType) + ;(ms as Mutable).type = fnType + + addParamNames(fnType, n) + addNamespaceToType(fnType) + } + } + + return + } + + if (n.kind === 'FieldDecl' && n.name) { + const d = (n as FieldDeclNode) + const s = getContainerSymbol(scopes[scopes.length - 1].name) + if (s && (s.decl.kind === 'CXXRecordDecl' || s.decl.kind === 'ClassTemplateDecl' || s.decl.kind === 'RecordDecl')) { + const ms = s.members[d.name] = createSymbol(n) + const x = d.type.qualType.match(/\(unnamed (union|struct) at (.+):([0-9]+):([0-9]+)\)/) + if (x && prevNode?.kind === 'RecordDecl' && (prevNode as any).completeDefinition) { + const r = prevNode as RecordDeclNode + const members: { name: string; type: CxxType }[] = [] + for (const z of r.inner ?? []) { + if (z.kind === 'FieldDecl' && z.name) { + members.push({ + name: z.name, + type: parseTypeStr((z as FieldDeclNode).type.desugaredQualType ?? (z as FieldDeclNode).type.qualType), + }) + } + } + + ;(ms as Mutable).type = { + kind: 'record', + variant: r.tagUsed === 'union' ? 'union' : undefined, + members, + } + + addNamespaceToType(ms.type!) + + return + } + + //const type = parseTypeStr(d.type.desugaredQualType ?? d.type.qualType) + const type = parseTypeStr(d.type.desugaredQualType ?? d.type.qualType) + ;(ms as Mutable).type = type + + addNamespaceToType(type) + } + + return + } + + function createBuiltinSymbol(r: ClangAstNode) { + const sym = symbols[r.id] + if (sym) { + return sym + } + + const ty = ((r as any).type as Type).qualType + const s = createSymbol({ ...r, name: ty }, ty) + ;(s as Mutable).type = { kind: 'ref', name: ty } + getGlobalNamespace()[ty] = s + + return s + } + + function getDeclSym(d: { id: string; name: string }) { + const sym = symbols[d.id] + if (sym) { + return sym + } + } + + if (n.kind === 'AccessSpecDecl' && visibility.length > 0) { + visibility[visibility.length - 1] = (n as any).access + + return + } + + if (n.kind === 'TypedefDecl') { + const ty = (n as { type?: Type }).type?.desugaredQualType + if (ty && n.name) { + if (n.inner && n.inner[0].kind === 'ElaboratedType') { + const r = n.inner[0] + if ((r as any).ownedTagDecl) { + const target = symbols[(r as any).ownedTagDecl.id] + if (target && target.name === `__anon_${target.id}`) { + ;(target as any).name = n.name + ;(target as any).fqn = [ + ...(target as any).fqn.split('::').slice(0, -1), + n.name, + ].join('::') + + getNamespace()[n.name] = target + symbols[n.id] = target + + return + } + + if (target?.name === n.name) { + return + } + + if (target && target.type) { + const s = createSymbol(n) + getNamespace()[n.name] = s + ;(s as Mutable).type = { + kind: 'ref', + name: target.name, + symbol: target, + } + + return + } + + const s = createSymbol(n) + getNamespace()[n.name] = s + ;(s as Mutable).type = { + kind: 'ref', + name: (r as any).ownedTagDecl.name, + } + + return + } + + if ((r as any).inner?.[0]?.kind === 'RecordType') { + const s = createSymbol(n) + getNamespace()[n.name] = s + ;(s as Mutable).type = { + kind: 'ref', + name: ty.replace(/^(struct|union) /, ''), + } + + return + } + } + + const s = createBuiltinSymbol(n) + + getNamespace()[n.name] = s + symbols[n.id] = s + + return + } + + if (n.name && n.inner) { + const r = n.inner[0] + if (r.kind === 'BuiltinType') { + const s = createBuiltinSymbol(r) + + getNamespace()[n.name] = s + symbols[n.id] = s + } else if (r.kind === 'ElaboratedType') { + + const r2 = r.inner?.[0] + if (r2?.kind === 'TypedefType' || r2?.kind === 'RecordType' || r2?.kind === 'EnumType') { + const d = (r2 as any).decl + const z = d ? symbols[d.id] : undefined + if (z) { + const sym = createSymbol(n) + ;(sym as Mutable).type = { kind: 'ref', name: d.name, symbol: z } + getNamespace()[n.name] = sym + + return + } + const s = getDeclSym(d) + if (s) { + getNamespace()[n.name] = s + }else { + getLogger().log('no symbol found inner', r2) + } + } else { + getLogger().log('unknown typedef inner', r2) + } + } else if (r.kind === 'RecordType') { + const d = (r as any).decl + const z = d ? symbols[d.id] : undefined + if (z) { + const sym = createSymbol(n) + ;(sym as Mutable).type = { kind: 'ref', name: d.name, symbol: z } + getNamespace()[n.name] = sym + + return + } + const s = d ? getDeclSym(d) : undefined + if (s) { + getNamespace()[n.name] = s + } else { + getLogger().log('no symbol found', r) + } + } else if (r.kind === 'PointerType' && (r as any).type) { + const sym = createSymbol(n) + ;(sym as Mutable).type = parseTypeStr((r as any).type.qualType) + getNamespace()[n.name] = sym + } else { + if (n.name === 'ssize_t') { + getLogger().log('unknown typedef', r) + } + } + } + + return + } + + // Note: external structs aren't relevant + if (n.kind === 'CXXRecordDecl' || n.kind === 'FunctionDecl' || n.kind === 'VarDecl' || n.kind === 'RecordDecl') { + const name = n.name ?? `__anon_${n.previousDecl ?? n.id}` + if (!n.name) { + // Add an anonymous symbol + // if (n.kind === 'CXXRecordDecl' || n.kind === 'RecordDecl') { + // createGlobalSymbol({ ...n, name: `__anon_${n.previousDecl ?? n.id}` }) + // } + prevNode = n + + // return + } + + const s = getSymbol(name) + if (s) { + // if (n.kind === 'CXXRecordDecl' && n.isImplicit && !(n as RecordDeclNode).completeDefinition && !n.inner) { + // return + // } + + while (currentAttrs.length > 0) { + s.attributes.push(currentAttrs.shift()!) + } + } + + if (s?.complete) { + return + } + + // Forward decl + if ((n.kind === 'CXXRecordDecl' || n.kind === 'RecordDecl') && !n.inner) { + return + } + + const ns = s ?? createGlobalSymbol(!n.name ? { ...n, name } : n) + if (n.kind === 'FunctionDecl' && (n as any).type) { + const ty = (n as any).type as Type + const fnType = parseFnType(ty.desugaredQualType ?? ty.qualType) + ;(ns as Mutable).type = fnType + + if (!s) { + addToParent(ns) + } + + addParamNames(fnType, n) + addNamespaceToType(fnType) + + return + } + + if (n.kind === 'VarDecl') { + return + } + + addToParent(ns) + visibility.push(undefined) + scopes.push({ type: 'record', name: ns.name }) + visitChildren(n) + scopes.pop() + visibility.pop() + + ;(ns as Mutable).complete = true + } + } + + const visited = new Set() + function visitSym(sym: Symbol) { + visited.add(sym) + + if (sym.type) { + addSymbols(sym.type) + } + + // Super hacky + if ((sym.decl as RecordDeclNode).bases) { + for (const b of (sym.decl as RecordDeclNode).bases!) { + try { + const t = Object.assign( + parseTypeStr(b.type.desugaredQualType ?? b.type.qualType), + { scopes: sym.fqn.split('::').slice(0, -1).map(x => ({ name: x, type: 'namespace' })) } + ) + addSymbols(t) + Object.assign(b, { __type: t }) + } catch {} + } + } + + for (const m of Object.values(sym.members)) { + if (!visited.has(m)) { + visitSym(m) + } + } + } + + // Re-map symbols to use FQN + const table: Record = {} + for (const sym of Object.values(symbols)) { + if (!visited.has(sym)) { + visitSym(sym) + } + table[sym.fqn] = sym + } + + return table +} + + +// clang++ -fno-exceptions -c + +// -vfsoverlay Overlay the virtual filesystem described by file over the real file system. Additionally, pass this overlay file to the linker if it supports it + +interface ParseOpt { + readonly include?: string[] + readonly mode?: 'c' | 'c++' +} + +export async function getAst(file: string, opt?: ParseOpt) { + const mode = opt?.mode ?? 'c++' + const args: string[] = [] + for (const dir of opt?.include ?? []) { + args.push('-I', `${dir}`) + } + + const executable = mode === 'c' ? 'clang' : 'clang++' + const res = await runCommand(executable, [...args, '-Xclang', '-ast-dump=json', file]) + const ast: AstRoot = JSON.parse(res) + + return ast +} + +export async function generateZigBindings(file: string) { + const ast = await getAst(file) + const symbols = resolveAst(ast, [file]) + const bindings = generateBindings(symbols) + console.log(bindings.cxxFile) + console.log('--------------') + console.log(bindings.zigFile) +} + +export async function generateTsBindings(file: string) { + const ast = await getAst(file, { mode: 'c', include: [path.dirname(file)] }) + const symbols = resolveAst(ast, [file]) + const bindings = _generateTsBindings(symbols) + + return bindings +} + +interface Type { + readonly qualType: string + readonly desugaredQualType?: string + readonly typeAliasDeclId?: string +} + +interface FunctionDeclNode extends ClangAstNode { + readonly kind: 'FunctionDecl' + readonly name: string + readonly type: Type + readonly inline?: boolean // CXX +} + +interface MethodDeclNode extends ClangAstNode { + readonly kind: 'CXXMethodDecl' + readonly name: string + readonly type: Type + readonly inline?: boolean // CXX + readonly virtual?: boolean + readonly pure?: boolean + readonly explicitlyDeleted?: boolean + readonly storageClass?: 'static' +} + +interface FieldDeclNode extends ClangAstNode { + readonly kind: 'FieldDecl' + readonly name: string + readonly type: Type + readonly storageClass?: 'static' +} + + + + +interface RecordBaseClause { + readonly access: 'public' | 'private' | 'protected' + readonly type: Type + readonly writtenAccess: 'none' +} + +// explicitlyDefaulted +// CXX +interface RecordDeclNode extends ClangAstNode { + readonly kind: 'CXXRecordDecl' + readonly tagUsed: 'struct' | 'union' | 'class' + readonly bases?: RecordBaseClause[] + readonly completeDefinition?: boolean + readonly definitionData?: { + isPOD?: boolean // deprecated + isTrivial?: boolean + isStandardLayout?: boolean + isAbstract?: boolean + isPolymorphic?: boolean + hasUserDeclaredConstructor?: boolean + defaultCtor: { + defaultedIsConstexpr?: boolean + exists?: boolean + isConstexpr?: boolean + needsImplicit?: boolean + trivial?: boolean + } + dtor: { + irrelevant?: boolean + needsImplicit?: boolean + simple?: boolean + trivial?: boolean + nonTrivial?: boolean + userDeclared?: boolean + } + } +} + diff --git a/src/zig/installer.ts b/src/zig/installer.ts new file mode 100644 index 0000000..855bca2 --- /dev/null +++ b/src/zig/installer.ts @@ -0,0 +1,229 @@ + +import * as path from 'node:path' +import * as builder from '../build/builder' +import { fetchData, fetchJson } from '../utils/http' +import { getPackageCacheDirectory, getUserSynapseDirectory, getWorkingDir } from '../workspaces' +import { runCommand } from '../utils/process' +import { ensureDir, isNonNullable, isWindows, memoize } from '../utils' +import { readKey, setKey } from '../cli/config' +import { extractToDir } from '../utils/tar' +import { registerToolProvider } from '../pm/tools' +import { getFs } from '../execution' +import { getLogger } from '..' + +interface ZigDownload { + readonly tarball: string // url + readonly shasum: string // sha256, hex + readonly size: number // bytes +} + +type Os = 'macos' | 'windows' | 'linux' | 'freebsd' +type Arch = 'x86_64' | 'i386' | 'aarch64' | 'riscv64' | 'armv7a' + +type ZigReleaseBase = { [P in PlatformPair]?: ZigDownload } + +interface ZigRelease extends ZigReleaseBase { + + // Other stuff that we don't use + readonly date: string // e.g. 2021-06-04 + readonly docs: string // url + readonly stdDocs: string // url + readonly notes: string // url + readonly src: ZigDownload + readonly bootstrap: ZigDownload + +} + +type PlatformPair = `${Arch}-${Os}` + +// signed with `minisign` +// key: RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U +// https://ziglang.org/download/0.11.0/zig-macos-aarch64-0.11.0.tar.xz.minisig +interface ZigVersionIndex { + readonly master: ZigRelease // This is built from the head commit + readonly [version: string]: ZigRelease +} + +function getOs(platform: typeof process.platform | 'windows' = process.platform): Os { + switch (platform) { + case 'darwin': + return 'macos' + + case 'win32': + case 'windows': + return 'windows' + + case 'linux': + case 'freebsd': + return platform + + default: + throw new Error(`Not supported: ${platform}`) + } +} + +function getArch(arch: typeof process.arch | 'aarch64' = process.arch): Arch { + switch (arch) { + case 'arm64': + return 'aarch64' + case 'x64': + return 'x86_64' + case 'ia32': + return 'i386' + case 'riscv64': + return 'riscv64' + case 'arm': + return 'armv7a' // not sure if this is right + + case 'aarch64': + return arch + + default: + throw new Error(`Not supported: ${arch}`) + } +} + +function toBuildArch(arch: Arch): builder.Arch | undefined { + switch (arch) { + case 'aarch64': + return arch + // case 'i386': + // return 'ia32' + // case 'riscv64': + // return 'riscv64' + case 'x86_64': + return 'x64' + } +} + +function toBuildOs(os: Os): builder.Os | undefined { + switch (os) { + case 'macos': + return 'darwin' + case 'windows': + case 'linux': + case 'freebsd': + return os + } +} + +async function getVersions() { + return await fetchJson('https://ziglang.org/download/index.json') +} + +function listOsArchPairs(release: ZigRelease) { + const pairs = Object.keys(release).map(x => { + const m = x.match(/^([a-z0-9_]+)-([a-z0-9_]+)$/) + if (!m) { + return + } + + const os = toBuildOs(m[2] as any) + if (!os) return + + const arch = toBuildArch(m[1] as any) + if (!arch) return + + return { os, arch } + }) + + return pairs.filter(isNonNullable) +} + +export function registerZigProvider() { + const getVersionsCached = memoize(getVersions) + registerToolProvider({ + name: 'zig', + listVersions: async () => { + const versions = await getVersionsCached() + + return Object.entries(versions).slice(1).flatMap(([k, v]) => listOsArchPairs(v).map(p => ({ ...p, version: k }))) as any + }, + getDownloadInfo: async release => { + const versions = await getVersionsCached() + const v = versions[release.version]?.[`${getArch(release.arch)}-${getOs(release.os)}`] + if (!v) { + throw new Error(`No such release found: ${release.os}-${release.arch}@${release.version}`) + } + + return { url: v.tarball, integrity: v.shasum } + }, + }) +} + +interface Target { + os?: Os + arch?: Arch + version?: string +} + +async function getReleaseDownload(target?: Target) { + const versions = await getVersions() + const os = target?.os ?? getOs() + const arch = target?.arch ?? getArch() + const version = target?.version ?? Object.keys(versions).slice(1)[0] // Assumes versions are sorted in descending order + + const release = versions[version]?.[`${arch}-${os}`] + if (!release) { + throw new Error(`No such release found: ${os}-${arch}@${version}`) + } + + return { + os, + arch, + version, + release, + } +} + +const getDefaultInstallDir = (version: string, os = toBuildOs(getOs()), arch = toBuildArch(getArch())) => { + return path.resolve(getPackageCacheDirectory(), 'synapse-tool', 'zig', `${os}-${arch}-${version}`) +} + +export async function installZig(dst?: string) { + const r = await getReleaseDownload() + const dir = dst ?? getDefaultInstallDir(r.version) + if (await getFs().fileExists(dir)) { + return dir + } + + await ensureDir(dir) + + const data = await fetchData(r.release.tarball) + const ext = path.extname(r.release.tarball) + + await extractToDir(data, dir, ext as '.xz' | '.zip') + + return dir +} + +async function getExecVersion(p: string) { + const resp = await runCommand(p, ['version']) + + return resp.trim() +} + +export async function getZigPath() { + const fromNodeModules = path.resolve(getWorkingDir(), 'node_modules', '.bin', isWindows() ? `zig.exe` : 'zig') + if (await getFs().fileExists(fromNodeModules)) { + return fromNodeModules + } + + const fromConfig = await readKey('zig.path') + if (fromConfig) { + return fromConfig + } + + const installDir = await installZig() + const zigPath = path.resolve(installDir, isWindows() ? `zig.exe` : 'zig') + // sanity check + const v = await getExecVersion(zigPath) + const expected = path.basename(installDir).split('-').at(-1) + if (v !== expected) { + throw new Error(`Unexpected zig version install: found ${v}, expected: ${expected}`) + } + + await setKey('zig.path', zigPath) + + return zigPath +} \ No newline at end of file diff --git a/src/zig/lib/js.zig b/src/zig/lib/js.zig new file mode 100644 index 0000000..0eef9ec --- /dev/null +++ b/src/zig/lib/js.zig @@ -0,0 +1,1197 @@ +const std = @import("std"); +const Type = std.builtin.Type; + + +extern fn napi_fatal_error(location: [*:0]const u8, location_len: usize, message: [*:0]const u8, message_len: usize) noreturn; + +pub const Value = opaque {}; + +const CallFrame = opaque { + extern fn napi_get_cb_info(env: *Env, frame: *CallFrame, argc: *usize, argv: [*]*Value, this: **Value, data: **anyopaque) Status; + + pub fn get_args(this: *CallFrame, env: *Env, comptime maxArgs: usize) !struct { [maxArgs]*Value, *Value, *anyopaque } { + var argc = maxArgs; + var argv: [maxArgs]*Value = undefined; + var this_arg: *Value = undefined; + var data: *anyopaque = undefined; + const status = napi_get_cb_info(env, this, &argc, &argv, &this_arg, &data); + if (status != .napi_ok) { + return error.NotOk; + } + + return .{ argv, this_arg, data }; + } +}; + +const Object = opaque { + extern fn napi_create_object(env: *Env, result: **Object) Status; + extern fn napi_set_named_property(env: *Env, object: *Object, name: [*:0]const u8, value: *Value) Status; + extern fn napi_define_properties(env: *Env, object: *Object, len: usize, descriptors: [*]PropertyDescriptor) Status; + + pub fn init(env: *Env) !*Object { + var result: *Object = undefined; + const status = napi_create_object(env, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + pub fn setNamedProperty(this: *Object, env: *Env, name: [:0]const u8, value: *Value) !void { + const status = napi_set_named_property(env, this, name, value); + if (status != .napi_ok) { + return error.NotOk; + } + } + + pub fn defineProperties(this: *Object, env: *Env, descriptors: []PropertyDescriptor) !void { + const status = napi_define_properties(env, this, descriptors.len, descriptors.ptr); + if (status != .napi_ok) { + return error.NotOk; + } + } +}; + +const Ref = opaque { + extern fn napi_create_reference(env: *Env, value: *Value, initial_refcount: u32, result: **Ref) Status; + + pub fn init(env: *Env, value: *Value, count: u32) !*Ref { + var result: *Ref = undefined; + const status = napi_create_reference(env, value, count, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } +}; + +const String = opaque { + extern fn napi_get_value_string_latin1(env: *Env, value: *String, buf: [*]u8, bufsize: usize, result: ?*usize) Status; + extern fn napi_get_value_string_utf8(env: *Env, value: *String, buf: ?[*]u8, bufsize: usize, result: ?*usize) Status; + extern fn napi_get_value_string_utf16(env: *Env, value: *String, buf: [*]u16, bufsize: usize, result: ?*usize) Status; + + extern fn napi_create_string_utf8(env: *Env, str: [*]const u8, len: usize, result: **String) Status; + + const FinalizeCb = fn (env: *Env, data: *anyopaque, hint: ?*anyopaque) void; + // `cb` -> `FinalizeCb` + extern fn node_api_create_external_string_latin1(env: *Env, str: [*]const u8, len: usize, cb: *const anyopaque, hint: ?*anyopaque, result: **String, copied: *bool) Status; + + pub fn fromUtf8(env: *Env, str: []const u8) !*String { + var result: *String = undefined; + const status = napi_create_string_utf8(env, str.ptr, str.len, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + fn deleteExternal(env: *Env, data: *anyopaque, hint: ?*anyopaque) void { + _ = env; + _ = data; + _ = hint; + } + + pub fn fromLatin1External(env: *Env, str: []const u8) !*String { + var result: *String = undefined; + var copied: bool = undefined; + const status = node_api_create_external_string_latin1(env, str.ptr, str.len, &deleteExternal, null, &result, &copied); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + pub fn toUtf8Buf(this: *String, env: *Env, buf: []u8) !usize { + var size = buf.len; + const status = napi_get_value_string_utf8(env, this, buf.ptr, size, &size); + if (status != .napi_ok) { + return error.NotOk; + } + + return size; + } + + pub fn toUtf8(this: *String, env: *Env, allocator: std.mem.Allocator) ![:0]u8 { + // Takes an extra call to get the length... + var size: usize = 0; + var status = napi_get_value_string_utf8(env, this, null, size, &size); + if (status != .napi_ok) { + return error.NotOk; + } + + var buf = try allocator.alloc(u8, size + 1); + status = napi_get_value_string_utf8(env, this, buf.ptr, size+1, null); + if (status != .napi_ok) { + return error.NotOk; + } + + // res[size] = 0; + + return buf[0..size :0]; + } +}; + +const Number = opaque { + // NAPI_EXTERN napi_status NAPI_CDECL napi_get_value_double(napi_env env, + // napi_value value, + // double* result); + // NAPI_EXTERN napi_status NAPI_CDECL napi_get_value_int32(napi_env env, + // napi_value value, + // int32_t* result); + // NAPI_EXTERN napi_status NAPI_CDECL napi_get_value_int64(napi_env env, + // napi_value value, + // int64_t* result); + // NAPI_EXTERN napi_status NAPI_CDECL napi_create_double(napi_env env, + // double value, + // napi_value* result); + // NAPI_EXTERN napi_status NAPI_CDECL napi_create_int32(napi_env env, + // int32_t value, + // napi_value* result); + // NAPI_EXTERN napi_status NAPI_CDECL napi_create_uint32(napi_env env, + // uint32_t value, + // napi_value* result); + // NAPI_EXTERN napi_status NAPI_CDECL napi_create_int64(napi_env env, + // int64_t value, + // napi_value* result); + + extern fn napi_get_value_uint32(env: *Env, value: *Number, result: *u32) Status; + extern fn napi_create_uint32(env: *Env, value: u32, result: **Number) Status; + + pub fn createU32(env: *Env, val: u32) !*Number { + var result: *Number = undefined; + const status = napi_create_uint32(env, val, &result); + if (status != .napi_ok) { + return error.NotOk; + } + return result; + } + + pub fn toU32(this: *Number, env: *Env) !u32 { + var val: u32 = undefined; + const status = napi_get_value_uint32(env, this, &val); + if (status != .napi_ok) { + return error.NotOk; + } + return val; + } +}; + +const PropertyAttributes = enum(u32) { + napi_default = 0, + napi_writable = 1 << 0, + napi_enumerable = 1 << 1, + napi_configurable = 1 << 2, + + // Used with napi_define_class to distinguish static properties + // from instance properties. Ignored by napi_define_properties. + napi_static = 1 << 10, + + // Default for class methods. + napi_default_method = .napi_writable | .napi_configurable, + + // Default for object properties, like in JS obj[prop]. + napi_default_jsproperty = .napi_writable | + .napi_enumerable | + .napi_configurable, +}; + +const PropertyDescriptor = extern struct { + // One of utf8name or name should be NULL. + utf8name: [:0]const u8, + name: *Value, + + method: *NativeFn, + getter: *NativeFn, + setter: *NativeFn, + value: *Value, + + attributes: PropertyAttributes, + data: *anyopaque, +}; + +const Function = opaque { + extern fn napi_create_function(env: *Env, name: [*]const u8, len: usize, cb: *const anyopaque, data: ?*anyopaque, result: **Function) Status; + extern fn napi_create_fastcall_function(env: *Env, name: [*]const u8, len: usize, cb: *const anyopaque, fast_cb: *CFunction, data: ?*anyopaque, result: **Function) Status; + + pub fn init(env: *Env, name: []const u8, cb: *const NativeFn) !*Function { + var result: *Function = undefined; + const status = napi_create_function(env, name.ptr, name.len, cb, null, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + pub fn initFastcall(env: *Env, name: []const u8, cb: *const NativeFn, fast_cb: *CFunction) !*Function { + var result: *Function = undefined; + const status = napi_create_fastcall_function(env, name.ptr, name.len, cb, fast_cb, null, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } +}; + +const CFunction = opaque { + const CType = enum(u32) { + void, + bool, + uint8, + int32, + uint32, + int64, + uint64, + float32, + float64, + pointer, + v8_value, + seq_one_byte_string, + api_obj, + any, + }; + + const CTypeDef = extern struct { + ty: CType, + sequence_type: c_uint = 0, + flags: c_uint = 0, + }; + + const CFunctionDef = extern struct { + return_type: CTypeDef, + args: [*]const CTypeDef, + arg_count: u32, + uses_options: bool, + }; + + extern fn napi_create_cfunction(def: *const CFunctionDef, cb: *const anyopaque, result: **CFunction) Status; + + pub fn init(def: *const CFunctionDef, cb: *const anyopaque) !*CFunction { + var result: *CFunction = undefined; + const status = napi_create_cfunction(def, cb, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } +}; + +// NAPI_EXTERN napi_status NAPI_CDECL napi_get_last_error_info( +// node_api_nogc_env env, const napi_extended_error_info** result); + +const ArrayPointer = opaque { + extern fn napi_set_element(env: *Env, object: *ArrayPointer, index: u32, val: *Value) Status; + extern fn napi_has_element(env: *Env, object: *ArrayPointer, index: u32, result: *bool) Status; + extern fn napi_get_element(env: *Env, object: *ArrayPointer, index: u32, result: **Value) Status; + extern fn napi_delete_element(env: *Env, object: *ArrayPointer, index: u32, result: *bool) Status; + extern fn napi_get_array_length(env: *Env, object: *ArrayPointer, result: *u32) Status; + + const IterateResult = enum(u8) { + _exception, + _break, + _continue, + }; + + const IterateCb = fn (index: u32, element: *Value, data: *anyopaque) IterateResult; + + extern fn napi_iterate(env: *Env, object: *ArrayPointer, cb: *const IterateCb, data: *anyopaque) Status; + + pub fn set(this: *ArrayPointer, env: *Env, index: u32, val: *Value) !void { + const status = napi_set_element(env, this, index, val); + if (status != .napi_ok) { + return error.NotOk; + } + } + + pub fn get(this: *ArrayPointer, env: *Env, index: u32) !*Value { + var result: *Value = undefined; + const status = napi_get_element(env, this, index, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + pub fn length(this: *ArrayPointer, env: *Env) !u32 { + var result: u32 = undefined; + const status = napi_get_array_length(env, this, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + fn copyTo(index: u32, element: *Value, data: *anyopaque) IterateResult { + var buf: []*Value = @ptrCast(data); + buf[index] = element; + return ._continue; + } + + pub fn copy(this: *ArrayPointer, env: *Env, allocator: std.mem.Allocator) ![]*Value { + const len = try this.length(env); + if (len == 0) { + return &[_]*Value{}; + } + + var buf = try allocator.alloc(*Value, len); + for (0..len) |i| { + buf[i] = try this.get(env, @intCast(i)); + } + // const status = napi_iterate(env, this, &ArrayPointer.copyTo, buf); + // if (status != .napi_ok) { + // return error.NotOk; + // } + + return buf; + } +}; + +// fn Array(comptime T: type) type {} + +const External = opaque { + const FinalizeCb = fn (env: *Env, data: *anyopaque, hint: ?*anyopaque) void; + + extern fn napi_create_external(env: *Env, data: *anyopaque, finalize_cb: ?*const FinalizeCb, finalize_hint: ?*anyopaque, result: **External) Status; + extern fn napi_get_value_external(env: *Env, value: *External, result: **anyopaque) Status; +}; + +const ArrayBuffer = opaque {}; + +const ModuleInit = fn (env: *Env, exports: *Object) void; + +const Module = opaque { + const ModuleDesc = extern struct { + nm_version: u32, + nm_flags: u32, + nm_filename: [:0]const u8, + nm_register_func: *ModuleInit, + nm_modname: [:0]const u8, + nm_priv: *anyopaque, + reserved: [4]*anyopaque, + }; + + extern fn napi_module_register(mod: *ModuleDesc) void; +}; + +const FastFunc = struct { + cb: *const anyopaque, + def: *const CFunction.CFunctionDef, +}; + +const FnDecl = struct { + name: []const u8, + cb: *const NativeFn, + fast_call_def: ?FastFunc, +}; + +fn MakeTuple(comptime t: Type.Fn) type { + var fields: [t.params.len]Type.StructField = undefined; + const z = Type{ .Struct = Type.Struct{ + .is_tuple = true, + .layout = .auto, + .fields = &fields, + .decls = &[0]Type.Declaration{}, + } }; + comptime var x = 0; + for (t.params) |a| { + var buf: [32]u8 = undefined; + const zz = std.fmt.formatIntBuf(&buf, x, 10, .lower, .{}); + buf[zz] = 0; + fields[x] = Type.StructField{ + .name = buf[0..zz :0], + .type = a.type orelse unreachable, + .default_value = null, + .is_comptime = false, + .alignment = @alignOf(a.type orelse unreachable), + }; + x += 1; + } + return @Type(z); +} + +fn toCTypeDef(comptime T: type) ?CFunction.CTypeDef { + return switch (T) { + void => CFunction.CTypeDef{ .ty = .void }, + u32 => CFunction.CTypeDef{ .ty = .uint32 }, + *Value => CFunction.CTypeDef{ .ty = .v8_value }, + *Receiver => CFunction.CTypeDef{ .ty = .v8_value }, + else => null, + }; +} + +fn makeFastcallFnDef(comptime t: Type.Fn) ?CFunction.CFunctionDef { + var args: [t.params.len]CFunction.CTypeDef = undefined; + inline for (t.params, 0..t.params.len) |p, i| { + args[i] = toCTypeDef(p.type orelse unreachable) orelse return null; + } + + const final = args[0..args.len].*; + + return .{ + .return_type = toCTypeDef(t.return_type orelse unreachable) orelse return null, + .args = &final, + .arg_count = t.params.len, + .uses_options = false, + }; +} + +fn makeFn(comptime t: Type.Fn, comptime v: anytype) NativeFn { + const Args = MakeTuple(t); + const ReturnType = t.return_type orelse unreachable; + const FnAsyncTypes = getAsyncTypes(ReturnType); + + if (FnAsyncTypes != null) { + return struct { + fn cb(env: *Env, info: *CallFrame) ?*Value { + return @ptrCast(runAsyncFn(v, env, info) catch return null); + } + }.cb; + } + + const s = struct { + fn cb(env: *Env, info: *CallFrame) ?*Value { + const wrap = EnvWrap.getWrap(env) catch return null; + var converter = wrap.initConverter(); + defer converter.deinit(); + + const z = info.get_args(env, t.params.len) catch return null; + var args: Args = undefined; + if (t.params.len > 0 and t.params[0].type == *Receiver) { + args[0] = converter.fromJs(t.params[0].type orelse unreachable, z[1]) catch return null; + inline for (z[0][0 .. t.params.len - 1], 1..t.params.len) |x, i| { + args[i] = converter.fromJs(t.params[i].type orelse unreachable, x) catch return null; + } + } else { + inline for (z[0], 0..t.params.len) |x, i| { + args[i] = converter.fromJs(t.params[i].type orelse unreachable, x) catch return null; + } + } + + const ret = @call(.always_inline, v, args); + + return switch (t.return_type orelse unreachable) { + *Value => ret, + u32 => @ptrCast(Number.createU32(env, ret) catch return null), + else => null, + }; + } + }; + + return s.cb; +} + +const HandleScope = opaque { + extern fn napi_open_handle_scope(env: *Env, result: **HandleScope) Status; + extern fn napi_close_handle_scope(env: *Env, scope: *HandleScope) Status; + + pub fn init(env: *Env) !*HandleScope { + var result: *HandleScope = undefined; + const status = napi_open_handle_scope(env, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + pub fn deinit(this: *HandleScope, env: *Env) !void { + const status = napi_close_handle_scope(env, this); + if (status != .napi_ok) { + return error.NotOk; + } + } +}; + +var currentEnv: *Env = undefined; +fn MakeInit(comptime decls: []const FnDecl) type { + return struct { + fn init(env: *Env, exports: *Object) callconv(.C) ?*Object { + currentEnv = env; + + const wrap = EnvWrap.getWrap(env) catch return null; + //var scope = HandleScope.init(env) catch return null; + + inline for (decls) |d| { + if (d.fast_call_def) |s| { + const fast_cb = CFunction.init(s.def, s.cb) catch return null; + const zz = Function.initFastcall(env, d.name, d.cb, fast_cb) catch return null; + const n: [:0]u8 = wrap.allocator.allocSentinel(u8, d.name.len, 0) catch return null; + @memcpy(n, d.name); + exports.setNamedProperty(env, n, @ptrCast(zz)) catch {}; + } else { + const zz = Function.init(env, d.name, d.cb) catch return null; + const n: [:0]u8 = wrap.allocator.allocSentinel(u8, d.name.len, 0) catch return null; + @memcpy(n, d.name); + exports.setNamedProperty(env, n, @ptrCast(zz)) catch {}; + } + } + + //scope.deinit(env) catch {}; + + return null; + } + }; +} + +pub fn registerModule(comptime T: type) void { + comptime var numDecls = 0; + comptime var decls: [256]FnDecl = undefined; + const info = @typeInfo(T); + + switch (info) { + .Struct => |s| { + inline for (s.decls) |decl| { + const val = @field(T, decl.name); + const t = @typeInfo(@TypeOf(val)); + + switch (t) { + .Fn => |f| { + const S = struct { + const fn_def = makeFastcallFnDef(f); + }; + const d = FnDecl{ + .name = decl.name, + .cb = &makeFn(f, val), + .fast_call_def = if (S.fn_def) |d| .{ .cb = &val, .def = &d } else null, + }; + decls[numDecls] = d; + numDecls += 1; + }, + else => {}, + } + } + }, + else => {}, + } + + const final = decls[0..numDecls].*; + const initStruct = MakeInit(&final); + @export(initStruct.init, .{ .name = "napi_register_module_v1", .linkage = .strong }); +} + +const EscapableHandleScope = opaque { + extern fn napi_open_escapable_handle_scope(env: *Env, result: **EscapableHandleScope) Status; + extern fn napi_close_escapable_handle_scope(env: *Env, scope: *EscapableHandleScope) Status; + extern fn napi_escape_handle(env: *Env, scope: *EscapableHandleScope, escapee: *Value, result: **Value) Status; + + pub fn open(env: *Env) !*EscapableHandleScope { + var result: *EscapableHandleScope = undefined; + const status = napi_open_escapable_handle_scope(env, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + pub fn close(this: *EscapableHandleScope, env: *Env) !void { + const status = napi_close_escapable_handle_scope(env, this); + if (status != .napi_ok) { + return error.NotOk; + } + } + + pub fn escape(this: *EscapableHandleScope, env: *Env, escapee: *Value) !*Value { + var result: *Value = undefined; + const status = napi_escape_handle(env, this, escapee, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } +}; + +var _task_name: ?*String = null; +const task_name = "task"; + +const AsyncTask = opaque { + const Execute = *const fn (*Env, ?*anyopaque) void; + const Complete = *const fn (*Env, Status, ?*anyopaque) void; + + extern fn napi_create_async_work( + env: *Env, + resource: ?*Value, + name: ?*Value, + exec: *const anyopaque, // Execute + complete: *const anyopaque, // Complete + data: ?*anyopaque, + result: **AsyncTask, + ) Status; + + extern fn napi_queue_async_work(env: *Env, task: *AsyncTask) Status; + extern fn napi_cancel_async_work(env: *Env, task: *AsyncTask) Status; + extern fn napi_delete_async_work(env: *Env, task: *AsyncTask) Status; + + fn get_name(env: *Env) !*String { + if (_task_name) |n| { + return n; + } + + const scope = try EscapableHandleScope.open(env); + var name = try String.fromLatin1External(env, task_name); + name = @ptrCast(try scope.escape(env, @ptrCast(name))); + try scope.close(env); + const ref = try Ref.init(env, @ptrCast(name), 1); + _ = ref; + _task_name = name; + return name; + } + + pub fn init(env: *Env, exec: Execute, complete: Complete, data: ?*anyopaque) !*AsyncTask { + var result: *AsyncTask = undefined; + const name = try String.fromLatin1External(env, task_name); + const status = napi_create_async_work(env, null, @ptrCast(name), exec, complete, data, &result); + if (status != .napi_ok) { + return error.NotOk; + } + + return result; + } + + pub fn start(this: *AsyncTask, env: *Env) !void { + const status = napi_queue_async_work(env, this); + if (status != .napi_ok) { + return error.NotOk; + } + } + + pub fn cancel(this: *AsyncTask, env: *Env) !void { + const status = napi_cancel_async_work(env, this); + if (status != .napi_ok) { + return error.NotOk; + } + } + + pub fn deinit(this: *AsyncTask, env: *Env) !void { + const status = napi_delete_async_work(env, this); + if (status != .napi_ok) { + return error.NotOk; + } + } +}; + +inline fn fatal(src: std.builtin.SourceLocation, msg: [:0]const u8) noreturn { + var buf: [256]u8 = undefined; + const location = std.fmt.bufPrintZ(&buf, "{s}:{d}:{d}", .{src.file, src.line, src.column}) catch unreachable; + napi_fatal_error(location.ptr, location.len, msg.ptr, msg.len); +} + +fn toValue(comptime T: type, env: *Env, val: T) ?*Value { + return switch (T) { + void => null, + anyerror => { + // const n = @intFromError(val); + // return @ptrCast(Number.createU32(env, n) catch unreachable); + return @ptrCast(env.createError(val, "Native error") catch fatal(@src(), "Failed to create error")); + }, + u32 => @ptrCast(Number.createU32(env, val) catch fatal(@src(), "Failed to create number")), + *PromiseValue => @ptrCast(val), + *Value => val, + else => fatal(@src(), "Invalid type"), + }; +} + +// const TaskQueue = struct { +// pending: []*AsyncTask = &[_]*AsyncTask{}, +// pendingCount: u32 = 0, +// didInit: bool = false, + +// pub fn init() TaskQueue { +// return TaskQueue{}; +// } + +// fn grow(this: *TaskQueue) !void { +// if (!this.didInit) { +// this.didInit = true; +// const buf = try std.heap.c_allocator.alloc(*AsyncTask, 4); +// this.pending = buf; +// return; +// } + +// const buf = try std.heap.c_allocator.alloc(*AsyncTask, this.pending.len * 2); +// @memcpy(buf, this.pending[0..this.pendingCount]); +// std.heap.c_allocator.free(this.pending); +// this.pending = buf; +// } + +// pub fn push(this: *TaskQueue, item: *AsyncTask) !void { +// if (this.pendingCount + 1 >= this.pending.len) { +// try this.grow(); +// } +// this.pending[this.pendingCount] = item; +// this.pendingCount += 1; +// } + +// pub fn pop(this: *TaskQueue) ?*AsyncTask { +// if (this.pendingCount == 0) { +// return null; +// } + +// const val = this.pending[this.pendingCount - 1]; +// this.pendingCount -= 1; + +// return val; +// } +// }; + +// var tasks = TaskQueue.init(); +// var runningCount: u32 = 0; + +fn runAsyncFn(comptime func: anytype, env: *Env, info: *CallFrame) !*PromiseValue { + const Func = AsyncFn(func); + const frame = try Func.init(env, info); + const task = try AsyncTask.init(env, &Func.run, &Func.complete, frame); + frame.task = task; + // if (runningCount >= 8) { + // try tasks.push(task); + // return frame.wrap.promise; + // } + + try task.start(env); + // runningCount += 1; + + return frame.wrap.promise; +} + +fn getFnType(comptime val: anytype) Type.Fn { + return switch (@typeInfo(@TypeOf(val))) { + .Fn => |f| f, + else => unreachable, + }; +} + +/// Functions that return a Promise will appear asychronous to +/// JavaScript callers by running the function in a separate thread. +pub fn Promise(comptime T: type, comptime E: type) type { + return union(enum) { + resolved: T, + rejected: E, + + pub fn resolve(val: T) @This() { + return .{ .resolved = val }; + } + + pub fn reject(err: E) @This() { + return .{ .rejected = err }; + } + }; +} + +const AsyncTypes = struct { + resolved: type, + rejected: type, +}; + +fn getAsyncTypes(comptime t: type) ?AsyncTypes { + const info = @typeInfo(t); + const u = switch (info) { + .Union => |s| s, + else => return null, + }; + + if (!@hasField(t, "resolved") and !@hasField(t, "rejected")) return null; + + return .{ + .resolved = u.fields[0].type, + .rejected = u.fields[1].type, + }; +} + +fn getArrayElementType(comptime T: type) ?type { + const info = @typeInfo(T); + const s = switch (info) { + .Struct => |s| s, + else => return null, + }; + + if (!@hasField(T, "elements")) return null; + + return switch (@typeInfo(s.fields[0].type)) { + .Pointer => |a| a.child, + else => null, + }; +} + +// Current overhead (on my machine): ~2500ns (possibly ~1900ns) +fn AsyncFn(comptime func: anytype) type { + const funcType = getFnType(func); + const Args = MakeTuple(funcType); + const ReturnType = funcType.return_type orelse unreachable; + const Frame = struct { + args: Args, + ret: ReturnType, + wrap: PromiseWrap, + task: *AsyncTask, // Assigned by the caller + converter: ValueConverter, + + pub fn init(env: *Env, info: *CallFrame) !*@This() { + const wrap = try PromiseWrap.init(env); + const z = try info.get_args(env, funcType.params.len); + var args: Args = undefined; + var converter = wrap.envWrap.initConverter(); + inline for (z[0], 0..funcType.params.len) |x, i| { + args[i] = try converter.fromJs(funcType.params[i].type orelse unreachable, x); + } + + var this = try converter.arena.allocator().create(@This()); + this.args = args; + this.wrap = wrap; + this.converter = converter; + + return this; + } + + pub fn run(_: *Env, data: ?*anyopaque) void { + var this: *@This() = @alignCast(@ptrCast(data orelse unreachable)); + this.ret = @call(.auto, func, this.args); + } + + pub fn complete(env: *Env, _: Status, data: ?*anyopaque) void { + const this: *@This() = @alignCast(@ptrCast(data orelse unreachable)); + defer { + // runningCount -= 1; + // if (runningCount < 8) { + // if (tasks.pop()) |t| { + // t.start(env) catch {}; + // runningCount += 1; + // } + // } + + this.task.deinit(env) catch {}; + var conv = this.converter; + conv.deinit(); + } + + const promiseTypes = getAsyncTypes(ReturnType); + if (promiseTypes) |x| { + switch (this.ret) { + .resolved => |r| this.wrap.resolve(toValue(x.resolved, env, r)) catch {}, + .rejected => |e| this.wrap.reject(toValue(x.rejected, env, e)) catch {}, + } + } else { + this.wrap.resolve(toValue(ReturnType, env, this.ret)) catch {}; + } + } + }; + + return Frame; +} + +pub const UTF8String = struct { data: [:0]u8 }; + +const FunctionWrap = struct { + env: *Env, +}; + +const Env = opaque { + extern fn napi_create_error(env: *Env, code: *String, msg: *String, result: **Object) Status; + extern fn napi_get_undefined(env: *Env, result: **Value) Status; + extern fn napi_get_null(env: *Env, result: **Value) Status; + extern fn napi_get_boolean(env: *Env, val: bool, result: **Value) Status; + + pub fn getUndefined(this: *Env) !*Value { + var result: *Value = undefined; + const status = napi_get_undefined(this, &result); + if (status != .napi_ok) { + return error.NotOk; + } + return result; + } + + pub fn getNull(this: *Env) !*Value { + var result: *Value = undefined; + const status = napi_get_null(this, &result); + if (status != .napi_ok) { + return error.NotOk; + } + return result; + } + + pub fn getBool(this: *Env, val: bool) !*Value { + var result: *Value = undefined; + const status = napi_get_boolean(this, val, &result); + if (status != .napi_ok) { + return error.NotOk; + } + return result; + } + + pub fn createError(this: *Env, err: anyerror, msg: [:0]const u8) !*Object { + const errName = @errorName(err); + const msgString = try String.fromUtf8(this, msg); + const nameString = try String.fromUtf8(this, errName); + + var result: *Object = undefined; + const status = napi_create_error(this, nameString, msgString, &result); + if (status != .napi_ok) { + return error.NotOk; + } + return result; + } +}; + +const Status = enum(u16) { + napi_ok, + napi_invalid_arg, + napi_object_expected, + napi_string_expected, + napi_name_expected, + napi_function_expected, + napi_number_expected, + napi_boolean_expected, + napi_array_expected, + napi_generic_failure, + napi_pending_exception, + napi_cancelled, + napi_escape_called_twice, + napi_handle_scope_mismatch, + napi_callback_scope_mismatch, + napi_queue_full, + napi_closing, + napi_bigint_expected, + napi_date_expected, + napi_arraybuffer_expected, + napi_detachable_arraybuffer_expected, + napi_would_deadlock, + napi_no_external_buffers_allowed, + napi_cannot_run_js, +}; + +const NativeFn = fn (env: *Env, info: *CallFrame) ?*Value; + +const PromiseValue = opaque {}; +const Deferred = opaque { + extern fn napi_resolve_deferred(env: *Env, deferred: *Deferred, value: *Value) Status; + extern fn napi_reject_deferred(env: *Env, deferred: *Deferred, value: *Value) Status; + + pub fn resolve(this: *Deferred, env: *Env, value: *Value) !void { + const status = napi_resolve_deferred(env, this, value); + if (status != .napi_ok) { + return error.NotOk; + } + } + + pub fn reject(this: *Deferred, env: *Env, value: *Value) !void { + const status = napi_reject_deferred(env, this, value); + if (status != .napi_ok) { + return error.NotOk; + } + } +}; + +const ValueConverter = struct { + env: *Env, + arena: std.heap.ArenaAllocator, + + pub fn deinit(this: *ValueConverter) void { + this.arena.deinit(); + } + + pub fn fromJs(this: *ValueConverter, comptime T: type, val: *Value) !T { + if (T == UTF8String) { + const s: *String = @ptrCast(val); + const data: [:0]u8 = try s.toUtf8(this.env, this.arena.allocator()); + + return UTF8String{ .data = data }; + } + + if (getArrayElementType(T)) |U| { + const arr: *ArrayPointer = @ptrCast(val); + const tmp = try arr.copy(this.env, this.arena.allocator()); + var tmp2: []U = try this.arena.allocator().alloc(U, tmp.len); + for (tmp, 0..tmp.len) |x, i| { + tmp2[i] = try this.fromJs(U, x); + } + + return T{ .elements = tmp2 }; + } + + return switch (T) { + *Value => val, + *Receiver => @ptrCast(val), + *String => @ptrCast(val), + *Number => @ptrCast(val), + u32 => { + const n: *Number = @ptrCast(val); + return try n.toU32(this.env); + }, + else => unreachable, + }; + } +}; + +// I should probably expose this apart of the node binary rather than the addon api +extern fn napi_wait_for_promise(env: *Env, value: *Value, result: **Value) Status; + +pub fn waitForPromise(value: *Value) !*Value { + var result: *Value = undefined; + const status = napi_wait_for_promise(currentEnv, value, &result); + if (status != .napi_ok) { + return error.NotOk; + } + return result; +} + +const Allocator = std.heap.GeneralPurposeAllocator(.{}); +var gpa = Allocator{}; + +const EnvWrap = struct { + const Envs = std.AutoHashMap(*Env, EnvWrap); + var envs: ?Envs = null; + + fn initEnvs() !Envs { + if (envs) |o| { + return o; + } + + const m = Envs.init(gpa.allocator()); + envs = m; + return m; + } + + pub fn getWrap(env: *Env) !*EnvWrap { + var _envs = try initEnvs(); + var entry = try _envs.getOrPut(env); + if (!entry.found_existing) { + entry.value_ptr.env = env; + entry.value_ptr.allocator = gpa.allocator(); + } + + return entry.value_ptr; + } + + env: *Env, + allocator: std.mem.Allocator, + + _null: ?*Value = null, + _true: ?*Value = null, + _false: ?*Value = null, + _undefined: ?*Value = null, + + pub fn getNull(this: *EnvWrap) !*Value { + if (this._null) |v| { + return v; + } + + const result = try this.env.getNull(); + this._null = result; + return result; + } + + pub fn getUndefined(this: *EnvWrap) !*Value { + if (this._undefined) |v| { + return v; + } + + const result = try this.env.getUndefined(); + this._undefined = result; + return result; + } + + pub fn getTrue(this: *EnvWrap) !*Value { + if (this._true) |v| { + return v; + } + + const result = try this.env.getBool(true); + this._true = result; + return result; + } + + pub fn getFalse(this: *EnvWrap) !*Value { + if (this._false) |v| { + return v; + } + + const result = try this.env.getBool(false); + this._false = result; + return result; + } + + pub fn initConverter(this: *EnvWrap) ValueConverter { + return ValueConverter{ + .env = this.env, + .arena = std.heap.ArenaAllocator.init(this.allocator), + }; + } +}; + +const PromiseWrap = struct { + extern fn napi_create_promise(env: *Env, deferred: **Deferred, promise: **PromiseValue) Status; + + envWrap: *EnvWrap, + deferred: *Deferred, + promise: *PromiseValue, + + pub fn init(env: *Env) !PromiseWrap { + var deferred: *Deferred = undefined; + var promise: *PromiseValue = undefined; + const envWrap = try EnvWrap.getWrap(env); + const status = napi_create_promise(env, &deferred, &promise); + if (status != .napi_ok) { + return error.NotOk; + } + + return .{ + .envWrap = envWrap, + .deferred = deferred, + .promise = promise, + }; + } + + pub fn resolve(this: PromiseWrap, value: ?*Value) !void { + try this.deferred.resolve(this.envWrap.env, value orelse try this.envWrap.env.getUndefined()); + } + + pub fn reject(this: PromiseWrap, value: ?*Value) !void { + try this.deferred.reject(this.envWrap.env, value orelse try this.envWrap.env.getUndefined()); + } +}; + +pub fn Array(comptime T: type) type { + return struct { + elements: []T, + }; +} + +const Receiver = opaque {}; + +const TestModule = struct { + // pub fn addSync(this: *Receiver, a: u32, b: u32) u32 { + // // std.debug.print("{}", .{this}); + // _ = this; + // return a + b; + // } + + // pub fn add(a: u32, b: u32) Promise(u32, void) { + // return Promise(u32, void).resolve(a + b); + // } + + pub fn add2(a: u32, b: Array(u32)) Promise(void, void) { + std.debug.print("{}, {}", .{ a, b }); + return Promise(void, void).resolve({}); + } +}; + +// comptime { +// registerModule(TestModule); +// } + +// --allow-natives-syntax +// %CompileOptimized +// %PrepareFunctionForOptimization +// %OptimizeFunctionOnNextCall +// const status = %GetOptimizationStatus(doAdd) +// console.log('maybe deopt', (status >> 3) & 1) +// console.log('optimized', (status >> 4) & 1) +// console.log('magleved', (status >> 5) & 1) +// console.log('turbofanned', (status >> 6) & 1) \ No newline at end of file diff --git a/src/zig/lib/mem.zig b/src/zig/lib/mem.zig new file mode 100644 index 0000000..e6dc6d0 --- /dev/null +++ b/src/zig/lib/mem.zig @@ -0,0 +1,33 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const isWasm = builtin.target.isWasm(); +const Allocator = if (isWasm) std.heap.WasmPageAllocator else std.heap.GeneralPurposeAllocator(.{}); + +pub const allocator = if (isWasm) toAllocator(&Allocator{}) else (Allocator{}).allocator(); + +pub fn strlen(source: [*:0]const u8) usize { + var i: usize = 0; + while (source[i] != 0) i += 1; + return i; +} + +export fn memcpy(dest: [*]u8, src: [*]const u8, len: usize) [*]u8 { + for (0..len) |i| { + dest[i] = src[i]; + } + return dest; +} + +export fn memset(dest: [*]u8, fill: u8, count: usize) [*]u8 { + for (0..count) |i| { + dest[i] = fill; + } + return dest; +} + +fn toAllocator(a: *const std.heap.WasmPageAllocator) std.mem.Allocator { + return std.mem.Allocator{ + .ptr = @constCast(a), + .vtable = &std.heap.WasmPageAllocator.vtable, + }; +} diff --git a/src/zig/util.ts b/src/zig/util.ts new file mode 100644 index 0000000..f16d030 --- /dev/null +++ b/src/zig/util.ts @@ -0,0 +1,18 @@ +import * as util from './util.zig' + +// This will likely crash if called inside an immediate callback +export function waitForPromise(promise: Promise | T): T { + if (process.release.name !== 'node-synapse') { + throw new Error(`"waitForPromise" is not available in the current runtime`) + } + + if (promise instanceof Promise) { + return util.waitForPromise(promise) + } + + if (!!promise && typeof promise === 'object' && 'then' in promise) { + return util.waitForPromise(promise) + } + + return promise +} \ No newline at end of file diff --git a/src/zig/util.zig b/src/zig/util.zig new file mode 100644 index 0000000..865272c --- /dev/null +++ b/src/zig/util.zig @@ -0,0 +1,10 @@ +const js = @import("./lib/js.zig"); + +pub fn waitForPromise(p: *js.Value) *js.Value { + return js.waitForPromise(p) catch unreachable; +} + +comptime { + js.registerModule(@This()); +} + diff --git a/src/zig/win32/load-hook.zig b/src/zig/win32/load-hook.zig new file mode 100644 index 0000000..f4e0996 --- /dev/null +++ b/src/zig/win32/load-hook.zig @@ -0,0 +1,67 @@ +// ref: https://github.com/nodejs/node-gyp/blob/af876e10f01ea8e3fdfeee20dbee3f7138ccffd5/src/win_delay_load_hook.cc +// This file won't be useful until Zig has an equivalent to the MSVC `/delayload` switch + +const std = @import("std"); +const DWORD = std.os.windows.DWORD; +const FARPROC = std.os.windows.FARPROC; +const HMODULE = std.os.windows.HMODULE; +const LPCSTR = std.os.windows.LPCSTR; +const WINAPI = std.os.windows.WINAPI; +const eql = std.mem.eql; + + +const Event = enum(c_uint) { + dliStartProcessing, // used to bypass or note helper only + dliNotePreLoadLibrary, // called just before LoadLibrary, can + // override w/ new HMODULE return val + dliNotePreGetProcAddress, // called just before GetProcAddress, can + // override w/ new FARPROC return value + dliFailLoadLib, // failed to load library, fix it by + // returning a valid HMODULE + dliFailGetProc, // failed to get proc address, fix it by + // returning a valid FARPROC + dliNoteEndProcessing, // called after all processing is done, no + // bypass possible at this point except + // by longjmp()/throw()/RaiseException. +}; + + + +const ImgDelayDescr = opaque {}; +const DelayLoadProc = struct { + fImportByName: bool, + data: union { + szProcName: LPCSTR, + dwOrdinal: DWORD, + }, +}; + +const DelayLoadInfo = struct { + cb: DWORD, // size of structure + pidd: *ImgDelayDescr, // raw form of data (everything is there) + ppfn: *FARPROC, // points to address of function to load + szDll: LPCSTR, // name of dll + dlp: DelayLoadProc, // name or ordinal of procedure + hmodCur: HMODULE, // the hInstance of the library we have loaded + pfnCur: FARPROC, // the actual function that will be called + dwLastError: DWORD, // error received (if an error notification) +}; + + +extern "kernel32" fn GetModuleHandleA(lpModuleName: ?LPCSTR) callconv(WINAPI) ?HMODULE; + +export fn __pfnDliNotifyHook2(dliNotify: Event, pdli: *DelayLoadInfo) callconv(WINAPI) ?FARPROC { + if (dliNotify != .dliNotePreLoadLibrary) { + return null; + } + + std.debug.print("{s}\n", .{pdli.szDll}); + + if (!eql(u8, std.mem.sliceTo(pdli.szDll, 0), "node.exe")) { + return null; + } + + const h = GetModuleHandleA(null); + + return @ptrCast(h); +} diff --git a/src/zig/win32/shim.zig b/src/zig/win32/shim.zig new file mode 100644 index 0000000..712f491 --- /dev/null +++ b/src/zig/win32/shim.zig @@ -0,0 +1,343 @@ +// This is a minimal executable to act as a proxy for executing other +// executable-like things on Windows. +// +// The idea was inspired by Bun's "bunx" feature, though this implementation +// doesn't use a second file to store the target information. It's possible +// that Bun's implementation could be faster in practice depending on how +// Windows caches executable images from the filesystem. + +const std = @import("std"); +const windows = std.os.windows; + +const PayloadType = enum(u8) { + NativeExecutable, + Interpretted, +}; + +const Payload = struct { + payloadType: PayloadType, + + /// Absolute path to the file to be executed, utf-16le encoded + target: [:0]u16, + + /// [optional] + /// Absolute path to a runtime executable to start the target file, utf-16le encoded + runtime: ?[:0]u16, +}; + +fn parsePayload(buf: []u8) !Payload { + var payload: Payload = undefined; + payload.payloadType = switch (buf[0]) { + 0 => PayloadType.NativeExecutable, + 1 => PayloadType.Interpretted, + else => return error.UnknownPayloadType, + }; + + const rem: [*]u16 = @alignCast(@ptrCast(buf[1..].ptr)); + var end: usize = 0; + for (0..(buf.len / 2)) |i| { + if (rem[i] == 0) { + end = i; + break; + } + } + + if (end == 0) { + return error.UnterminatedString; + } + + payload.target = rem[0..end :0]; + payload.runtime = null; + + switch (payload.payloadType) { + .NativeExecutable => {}, + .Interpretted => { + payload.runtime = rem[(end+1)..(buf.len / 2) :0]; + }, + } + + return payload; +} + + +const HANDLE = windows.HANDLE; + +pub fn OpenFile(sub_path_w: []const u16) !HANDLE { + var result: HANDLE = undefined; + + const path_len_bytes: u16 = @truncate(sub_path_w.len * 2); + + var nt_name = windows.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @constCast(sub_path_w.ptr), + }; + var attr = windows.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(windows.OBJECT_ATTRIBUTES), + .RootDirectory = null, + .Attributes = 0, + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + var io: windows.IO_STATUS_BLOCK = undefined; + + // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT. + const flags: windows.ULONG = windows.FILE_SYNCHRONOUS_IO_NONALERT | windows.FILE_NON_DIRECTORY_FILE | windows.FILE_OPEN_REPARSE_POINT; + + const rc = windows.ntdll.NtCreateFile( + &result, + windows.GENERIC_READ | windows.SYNCHRONIZE, + &attr, + &io, + null, + windows.FILE_ATTRIBUTE_NORMAL, + 0, + windows.FILE_OPEN, + flags, + null, + 0, + ); + switch (rc) { + .SUCCESS => return result, + .OBJECT_NAME_INVALID => return error.BadPathName, + .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, + .BAD_NETWORK_PATH => return error.NetworkNotFound, // \\server was not found + .BAD_NETWORK_NAME => return error.NetworkNotFound, // \\server was found but \\server\share wasn't + .NO_MEDIA_IN_DEVICE => return error.NoDevice, + .INVALID_PARAMETER => unreachable, + .SHARING_VIOLATION => return error.AccessDenied, + .ACCESS_DENIED => return error.AccessDenied, + .PIPE_BUSY => return error.PipeBusy, + .OBJECT_PATH_SYNTAX_BAD => unreachable, + .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, + .FILE_IS_A_DIRECTORY => return error.IsDir, + .NOT_A_DIRECTORY => return error.NotDir, + .USER_MAPPED_FILE => return error.AccessDenied, + .INVALID_HANDLE => unreachable, + + .VIRUS_INFECTED, .VIRUS_DELETED => return error.AntivirusInterference, + else => unreachable, + } +} + +fn dprint(v: [:0]const u16) void { + var buf: [512]u8 = undefined; + const size = std.unicode.utf16LeToUtf8(&buf, v) catch return; + std.debug.print("{s}\n", .{buf[0..size]}); +} + +fn toExitCode(err: anyerror) u32 { + const n: u32 = @intFromError(err); + + return n | 0x20000000; // bit 29 is reserved for application error codes +} + +fn dumpErrorCodes() void { + const ty = @typeInfo(@typeInfo(@TypeOf(exec)).Fn.return_type.?); + const u = switch (ty) { + .ErrorUnion => |u| u, + else => return, + }; + + const ty2 = @typeInfo(u.error_set); + + const set = switch (ty2) { + .ErrorSet => |s| s orelse return, + else => return, + }; + + for (set, 0..set.len) |e, i| { + @compileLog(0x20000000 + i + 1, e); + } +} + +// comptime { +// dumpErrorCodes(); +// } + +const magic: u32 = 0x74617261; + +fn readSelf(buf: []u8) !u32 { + const teb: *std.os.windows.TEB = @call(.always_inline, std.os.windows.teb, .{}); + const peb = teb.ProcessEnvironmentBlock; + const image_path_unicode_string = &peb.ProcessParameters.ImagePathName; + const image_path_name = image_path_unicode_string.Buffer.?[0 .. image_path_unicode_string.Length / 2 :0]; + var prefixed: [2048]u16 = undefined; + prefixed[0] = '\\'; + prefixed[1] = '?'; + prefixed[2] = '?'; + prefixed[3] = '\\'; + @memcpy(prefixed[4..], image_path_name); + prefixed[image_path_name.len+4] = 0; + const prefixed_path_w = prefixed[0..image_path_name.len+4 :0]; + + const handle = try OpenFile(prefixed_path_w); + defer windows.CloseHandle(handle); + + var bytesRead: windows.DWORD = undefined; + const r = windows.kernel32.ReadFile(handle, buf.ptr, @truncate(buf.len), &bytesRead, null); + if (r == 0) { + const err = windows.kernel32.GetLastError(); + switch (err) { + .SUCCESS => {}, + else => { + return error.FailedRead; + } + } + } + + return bytesRead; +} + +fn exec() !u32 { + var buf2: [20000]u8 = undefined; + const bytesRead = try readSelf(&buf2); + const footer: u32 = @bitCast((buf2[(bytesRead-4)..bytesRead][0..@sizeOf(u32)].*)); + if (footer != magic) { + return error.InvalidFooter; + } + + const size: u32 = @bitCast((buf2[(bytesRead-8)..bytesRead-4][0..@sizeOf(u32)].*)); + const payload = buf2[(bytesRead-(size+8))..(bytesRead-8)]; + const parsed = try parsePayload(payload); + + const cmd_line_w = windows.kernel32.GetCommandLineW(); + var proc = ChildProcess { + .id = undefined, + .thread_handle = undefined, + .term = undefined, + }; + if (parsed.payloadType == .NativeExecutable) { + try spawnWindows(&proc, parsed.target, cmd_line_w, null); + try waitUnwrappedWindows(&proc); + } else { + if (parsed.runtime) |rt| { + var c: usize = 0; + var j: usize = 0; + var start: usize = 0; + while (cmd_line_w[j] != 0) { + if (cmd_line_w[j] == '"') { + if (c == 1) { + start = j + 1; + break; + } else { + c += 1; + } + } + j += 1; + } + + if (start == 0) { + return error.BadParse; + } + + const target = parsed.target; + var cmd_buf: [25000]u16 = undefined; + cmd_buf[0] = '"'; + @memcpy(cmd_buf[1..], rt); + cmd_buf[rt.len] = '"'; + const start2 = rt.len+2; + cmd_buf[start2-1] = ' '; + + cmd_buf[start2] = '"'; + @memcpy(cmd_buf[(start2+1)..], target); + cmd_buf[target.len+start2+1] = '"'; + + const start3 = target.len+start2+2; + + var i: usize = 0; + while (cmd_line_w[start+i] != 0) { + cmd_buf[start3+i] = cmd_line_w[start+i]; + i += 1; + } + + cmd_buf[start3+i] = 0; + + const cmd_line_w2 = cmd_buf[0..(start3+i) :0]; + + try spawnWindows(&proc, rt, cmd_line_w2, null); + try waitUnwrappedWindows(&proc); + } + } + + return proc.term; +} + +pub fn main() noreturn { + const code = exec() catch |e| toExitCode(e); + windows.kernel32.ExitProcess(code); +} + + +fn windowsCreateProcess(app_name: [*:0]u16, cmd_line: [*:0]u16, envp_ptr: ?[*]u16, cwd_ptr: ?[*:0]u16, lpStartupInfo: *windows.STARTUPINFOW, lpProcessInformation: *windows.PROCESS_INFORMATION) !void { + return windows.CreateProcessW( + app_name, + cmd_line, + null, + null, + windows.TRUE, + windows.CREATE_UNICODE_ENVIRONMENT, + @as(?*anyopaque, @ptrCast(envp_ptr)), + cwd_ptr, + lpStartupInfo, + lpProcessInformation, + ); +} + +fn spawnWindows(self: *ChildProcess, app_name_w: [*:0]u16, cmd_line_w: [*:0]u16, cwd_w: ?[*:0]u16) !void { + const g_hChildStd_IN_Rd = windows.GetStdHandle(windows.STD_INPUT_HANDLE) catch null; + const g_hChildStd_OUT_Wr = windows.GetStdHandle(windows.STD_OUTPUT_HANDLE) catch null; + const g_hChildStd_ERR_Wr = windows.GetStdHandle(windows.STD_ERROR_HANDLE) catch null; + + var siStartInfo = windows.STARTUPINFOW{ + .cb = @sizeOf(windows.STARTUPINFOW), + .hStdError = g_hChildStd_ERR_Wr, + .hStdOutput = g_hChildStd_OUT_Wr, + .hStdInput = g_hChildStd_IN_Rd, + .dwFlags = windows.STARTF_USESTDHANDLES, + + .lpReserved = null, + .lpDesktop = null, + .lpTitle = null, + .dwX = 0, + .dwY = 0, + .dwXSize = 0, + .dwYSize = 0, + .dwXCountChars = 0, + .dwYCountChars = 0, + .dwFillAttribute = 0, + .wShowWindow = 0, + .cbReserved2 = 0, + .lpReserved2 = null, + }; + + var piProcInfo: windows.PROCESS_INFORMATION = undefined; + + try windowsCreateProcess(app_name_w, cmd_line_w, null, cwd_w, &siStartInfo, &piProcInfo); + + self.id = piProcInfo.hProcess; + self.thread_handle = piProcInfo.hThread; +} + +const ChildProcess = struct { + id: HANDLE, + thread_handle: HANDLE, + term: u32, +}; + +fn waitUnwrappedWindows(self: *ChildProcess) !void { + const result = windows.WaitForSingleObjectEx(self.id, windows.INFINITE, false); + + var exit_code: windows.DWORD = undefined; + if (windows.kernel32.GetExitCodeProcess(self.id, &exit_code) == 0) { + return error.UnknownExit; + } else { + self.term = exit_code; + } + + windows.CloseHandle(self.id); + windows.CloseHandle(self.thread_handle); + return result; +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..0039353 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "resolveJsonModule": true, + "checkJs": false, + "declaration": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": ".", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "declarationMap": true, + "skipLibCheck": true, + "alwaysStrict": true, + "allowArbitraryExtensions": true, + }, + "include": ["src"], + "exclude": ["**/*.d.zig.ts"] + } + \ No newline at end of file