Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: use plain app id in key gen #960

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

chore: use plain app id in key gen #960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
75c1da4
use plain app id in key gen
juliusmarminge Sep 18, 2024
b281a5c
rm verifykey helper
juliusmarminge Sep 18, 2024
37a73ff
Merge branch 'main' into plain-app-id
juliusmarminge Oct 8, 2024
764ed1b
Merge branch 'main' into plain-app-id
juliusmarminge Nov 6, 2024
6376d33
rm
juliusmarminge Nov 6, 2024
badab68
rm unused test
juliusmarminge Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 4 additions & 204 deletions docs/src/app/(docs)/uploading-files/page.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,211 +96,11 @@ Once your backend adapter has received and validated the request, you will next
need to generate presigned URLs. First, generate some file keys for the files to
be uploaded.

To generate a file key, generate a [Sqids](https://sqids.org/) of your appId
with `{ minLength: 12 }`, then concatenate this with a file seed of your choice.
To generate a file key, concatenate your appId with a file seed of your choice.
The file seed can be anything you want, but it should be unique for each file,
as well as url safe. In this example, we include a base64 encoding to ensure the
file seed is url safe, but you can do this however you want.

Although we currently only offer a JavaScript SDK, here are some reference
implementations you can use to generate valid file keys. We also plan on making
this process easier in the future.

<Note>
If you struggle to implement it for your language, you can also request one
from the [`/v7/prepareUpload` REST
endpoint](/api-reference/openapi-spec#tag/default/POST/v7/prepareUpload). Keep
in mind that this adds extra latency to your uploads.
</Note>

<Tabs tabs={["JavaScript", "Python", "PHP", "Go"]}>
<Tab>

```ts
import * as Hash from "effect/Hash";
import SQIds, { defaultOptions } from "sqids";

// A simple function to shuffle the alphabet for the Sqids
function shuffle(str: string, seed: string) {
const chars = str.split("");
const seedNum = Hash.string(seed);

let temp: string;
let j: number;
for (let i = 0; i < chars.length; i++) {
j = ((seedNum % (i + 1)) + i) % chars.length;
temp = chars[i];
chars[i] = chars[j];
chars[j] = temp;
}

return chars.join("");
}

function generateKey(appId: string, fileSeed: string) {
// Hash and Encode the parts and apiKey as sqids
const alphabet = shuffle(defaultOptions.alphabet, appId);

const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([
Math.abs(Hash.string(appId)),
]);

// We use a base64 encoding here to ensure the file seed is url safe, but
// you can do this however you want
const encodedFileSeed = encodeBase64(fileSeed);

return `${encodedAppId}${encodedFileSeed}`;
}
```

</Tab>
<Tab>

```py
import math
import base64
from sqids import Sqids
from sqids.constants import DEFAULT_ALPHABET

def hash_string(s: str) -> int:
h = 5381
for char in reversed(s):
h = (h * 33) ^ ord(char)
# 32-bit integer overflow
h &= 0xFFFFFFFF
h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000)

# Convert to signed 32-bit integer
if h >= 0x80000000:
h -= 0x100000000

return h


def shuffle(string: str, seed: str) -> str:
chars = list(string)
seed_num = hash_string(seed)

for i in range(len(chars)):
j = int(math.fmod(math.fmod(seed_num, i + 1) + i, len(chars)))
chars[i], chars[j] = chars[j], chars[i]

return "".join(chars)


def generate_key(file_seed: str, app_id: str) -> str:
alphabet = shuffle(DEFAULT_ALPHABET, app_id)

encoded_app_id = Sqids(alphabet, min_length=12).encode(
[abs(hash_string(app_id))]
)

return encoded_app_id + file_seed
```

</Tab>
<Tab>

```php
use Sqids\Sqids;

function hash_string(string $string): int {
$h = 5381;
for ($i = strlen($string) - 1; $i >= 0; $i--) {
$char = $string[$i];
$h = ($h * 33) ^ ord($char);
// 32-bit integer overflow
$h &= 0xFFFFFFFF;
}
$h = ($h & 0xBFFFFFFF) | (($h >> 1) & 0x40000000);

// Convert to signed 32-bit integer
if ($h >= 0x80000000) {
$h -= 0x100000000;
}

return $h;
}

function shuffle_string(string $string, string $seed): string {
$chars = str_split($string);
$seed_num = hash_string($seed);

for ($i = 0; $i < count($chars); $i++) {
$j = (($seed_num % ($i + 1)) + $i) % count($chars);
[$chars[$i], $chars[$j]] = [$chars[$j], $chars[$i]];
}

return implode('', $chars);
}

function generate_key(string $file_seed, string $appId): string {
$alphabet = shuffle_string(Sqids::DEFAULT_ALPHABET, $appId);
$sqids = new Sqids($alphabet, 12);

$encodedAppId = $sqids->encode(
[abs(hash_string($appId))]
);

return $encodedAppId . base64_encode($file_seed);
}
```

</Tab>
<Tab>

```go
import (
"math"
"github.com/sqids/sqids-go"
)

func hashString(s string) int32 {
h := int64(5381)
for i := len(s) - 1; i >= 0; i-- {
h = (h * 33) ^ int64(s[i])
// 32-bit integer overflow
h &= 0xFFFFFFFF
}
h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000)

// Convert to signed 32-bit integer
if h >= 0x80000000 {
h -= 0x100000000
}

return int32(h)
}

func shuffle(input string, seed string) string {
chars := []rune(input)
seedNum := hashString(seed)

for i := 0; i < len(chars); i++ {
j := (int(seedNum)%(i+1) + i) % len(chars)
chars[i], chars[j] = chars[j], chars[i]
}

return string(chars)
}

func generateKey(fileSeed string, appId string) string {
alphabet := shuffle("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", appId)
s, _ := sqids.New(sqids.Options{
MinLength: 12,
Alphabet: alphabet,
})

encodedAppId, _ := s.Encode(
[]uint64{uint64(math.Abs(float64(hashString(appId))))},
)

return encodedAppId + fileSeed
}
```

</Tab>
</Tabs>
url safe and no less than 36 characters. In this example, we include a base64
encoding to ensure the file seed is url safe, but you can do this however you
want.

The URL, to which you will upload the file, will depend on your app's region.
You can find the list of regions in the
Expand Down
19 changes: 1 addition & 18 deletions packages/shared/src/crypto.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,28 +111,11 @@ export const generateKey = (
const encodedFileSeed = new SQIds({ alphabet, minLength: 36 }).encode([
Math.abs(Hash.string(hashParts)),
]);
const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([
Math.abs(Hash.string(appId)),
]);

// Concatenate them
return encodedAppId + encodedFileSeed;
return appId + encodedFileSeed;
juliusmarminge marked this conversation as resolved.
Show resolved Hide resolved
}).pipe(Micro.withTrace("generateKey"));

// Verify that the key was generated with the same appId
export const verifyKey = (key: string, appId: string) =>
Micro.sync(() => {
const alphabet = shuffle(defaultOptions.alphabet, appId);
const expectedPrefix = new SQIds({ alphabet, minLength: 12 }).encode([
Math.abs(Hash.string(appId)),
]);

return key.startsWith(expectedPrefix);
}).pipe(
Micro.withTrace("verifyKey"),
Micro.orElseSucceed(() => false),
);

export const generateSignedURL = (
url: string | URL,
secretKey: Redacted.Redacted<string>,
Expand Down
93 changes: 0 additions & 93 deletions packages/shared/test/crypto.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {
generateKey,
generateSignedURL,
signPayload,
verifyKey,
verifySignature,
} from "../src/crypto";

Expand Down Expand Up @@ -106,96 +105,4 @@ describe("key gen", () => {
expect(key).toBeTruthy();
}),
);

it.effect("verifies a key", () =>
Effect.gen(function* () {
const appI = "foo-123";
const key = yield* generateKey(
{
name: "foo.txt",
size: 123,
type: "text/plain",
lastModified: Date.now(),
},
appI,
);

const verified = yield* verifyKey(key, appI);
expect(verified).toBe(true);
}),
);

it.effect("doesn't verify a key with a bad appI", () =>
Effect.gen(function* () {
const appI = "foo-123";
const key = yield* generateKey(
{
name: "foo.txt",
size: 123,
type: "text/plain",
lastModified: Date.now(),
},
appI,
);

const verified = yield* verifyKey(key, "bad");
expect(verified).toBe(false);
}),
);

it.effect("doesn't verify a key with a bad key", () =>
Effect.gen(function* () {
const appId = "foo-123";
const key = yield* generateKey(
{
name: "foo.txt",
size: 123,
type: "text/plain",
lastModified: Date.now(),
},
appId,
);

const verified = yield* verifyKey("badseed" + key.substring(7), appId);
expect(verified).toBe(false);
}),
);

it.effect("verifies with a custom hash function", () =>
Effect.gen(function* () {
const appI = "foo-123";
const key = yield* generateKey(
{
name: "foo.txt",
size: 123,
type: "text/plain",
lastModified: Date.now(),
},
appI,
(file) => [file.name],
);

const verified = yield* verifyKey(key, appI);
expect(verified).toBe(true);
}),
);

it.effect("works even when there's nothing to seed", () =>
Effect.gen(function* () {
const appI = "foo-123";
const key = yield* generateKey(
{
name: "foo.txt",
size: 123,
type: "text/plain",
lastModified: Date.now(),
},
appI,
() => [],
);

const verified = yield* verifyKey(key, appI);
expect(verified).toBe(true);
}),
);
});
Loading