diff --git a/pr-preview/pr-19/print.html b/pr-preview/pr-19/print.html index 47c9de6..4361f02 100644 --- a/pr-preview/pr-19/print.html +++ b/pr-preview/pr-19/print.html @@ -892,13 +892,14 @@
0x6412c4cfbe50e219c2d4d30108d7321d064e15bf64e752307100bff5eb91da38
.
The steps that follow require that the registration object is owned by the address you are using on -the Sui CLI.
-To find the address, execute:
+The steps that follow require that the SuiNS registration object is owned by the address you are +using on the Sui CLI. Therefore, we need to send this registration object from the address you use +in your browser wallet, to the address of your Sui CLI.
+To find the Sui CLI address, execute:
sui client active-address
-Then, from your browser wallet, select "Assets", look for the NFT of the SuiNS registration, which -should look as follows:
+Then, from your browser wallet, select the "Assets" tab, and look for the NFT of the SuiNS +registration, which should look as follows:
Click on it, scroll down to "Send NFT", and send it to the address discovered with the command above. Now, your Sui CLI address owns the registration NFT, and you can proceed to the next step.
diff --git a/pr-preview/pr-19/searchindex.js b/pr-preview/pr-19/searchindex.js index 6bad1f9..a9a71f8 100644 --- a/pr-preview/pr-19/searchindex.js +++ b/pr-preview/pr-19/searchindex.js @@ -1 +1 @@ -Object.assign(window.search, {"doc_urls":["index.html#walrus","index.html#features","index.html#architecture-and-operations","index.html#sources","objectives_use_cases.html#objectives-and-use-cases","objectives_use_cases.html#non-objectives","objectives_use_cases.html#use-cases","overview.html#system-overview","properties.html#walrus-assurance-and-security-properties","architecture.html#basic-architecture-and-security-assumptions","encoding.html#encoding-overheads-and-verification","operations.html#operations","operations-sui.html#operations-on-sui","operations-sui.html#storage-resource-life-cycle-on-sui","operations-sui.html#governance-operations-on-sui","operations-off-chain.html#off-chain-operations","operations-off-chain.html#write-paths","operations-off-chain.html#refresh-availability","operations-off-chain.html#inconsistent-resource-flow","operations-off-chain.html#read-paths","operations-off-chain.html#challenge-mechanism-for-storage-attestation","future.html#future-discussion","interacting.html#interacting-with-walrus","client-cli.html#using-the-walrus-client","json-api.html#json-mode","web-api.html#client-daemon-mode--http-api","web-api.html#starting-the-daemon","web-api.html#http-api-usage","walrus-sites/intro.html#introduction-to-walrus-sites","walrus-sites/intro.html#show-me","walrus-sites/tutorial.html#your-first-walrus-site","walrus-sites/tutorial-install.html#installing-the-site-builder","walrus-sites/tutorial-install.html#prerequisites","walrus-sites/tutorial-install.html#get-the-walrus-binary-and-install-it","walrus-sites/tutorial-install.html#point-your-sui-cli-to-testnet-and-get-some-sui","walrus-sites/tutorial-install.html#clone-the-walrus-sites-repo-and-build-the-site-builder-tool","walrus-sites/tutorial-install.html#get-the-latest-walrus-client-configuration","walrus-sites/tutorial-publish.html#publishing-a-walrus-site","walrus-sites/tutorial-publish.html#select-the-source-material-for-the-site","walrus-sites/tutorial-publish.html#publish-the-site","walrus-sites/tutorial-publish.html#update-the-site","walrus-sites/tutorial-publish.html#additional-commands","walrus-sites/tutorial-suins.html#bonus-set-a-suins-name","walrus-sites/tutorial-suins.html#get-a-suins-name","walrus-sites/tutorial-suins.html#send-the-suins-registration-object-to-the-address-you-use-with-the-sui-cli","walrus-sites/tutorial-suins.html#map-the-suins-name-to-the-walrus-site","walrus-sites/tutorial-config.html#configuring-the-site-builder","walrus-sites/tutorial-config.html#minimal-configuration","walrus-sites/tutorial-config.html#advanced-configuration","walrus-sites/overview.html#overview","walrus-sites/site-builder.html#the-site-builder","walrus-sites/portal.html#the-walrus-sites-portal","walrus-sites/restrictions.html#known-restrictions","walrus-sites/restrictions.html#no-secret-values","walrus-sites/restrictions.html#there-is-a-maximum-redirect-depth","walrus-sites/restrictions.html#service-workers-are-not-available","walrus-sites/restrictions.html#the-ios-sui-wallet-mobile-does-not-work-with-walrus-sites","glossary.html#walrus-glossary"],"index":{"documentStore":{"docInfo":{"0":{"body":33,"breadcrumbs":2,"title":1},"1":{"body":132,"breadcrumbs":2,"title":1},"10":{"body":277,"breadcrumbs":5,"title":3},"11":{"body":16,"breadcrumbs":2,"title":1},"12":{"body":63,"breadcrumbs":5,"title":2},"13":{"body":235,"breadcrumbs":8,"title":5},"14":{"body":92,"breadcrumbs":6,"title":3},"15":{"body":11,"breadcrumbs":5,"title":2},"16":{"body":206,"breadcrumbs":5,"title":2},"17":{"body":34,"breadcrumbs":5,"title":2},"18":{"body":150,"breadcrumbs":6,"title":3},"19":{"body":95,"breadcrumbs":5,"title":2},"2":{"body":101,"breadcrumbs":3,"title":2},"20":{"body":106,"breadcrumbs":7,"title":4},"21":{"body":54,"breadcrumbs":4,"title":2},"22":{"body":27,"breadcrumbs":4,"title":2},"23":{"body":46,"breadcrumbs":8,"title":3},"24":{"body":80,"breadcrumbs":8,"title":2},"25":{"body":19,"breadcrumbs":11,"title":5},"26":{"body":82,"breadcrumbs":8,"title":2},"27":{"body":79,"breadcrumbs":9,"title":3},"28":{"body":128,"breadcrumbs":6,"title":3},"29":{"body":184,"breadcrumbs":4,"title":1},"3":{"body":16,"breadcrumbs":2,"title":1},"30":{"body":15,"breadcrumbs":6,"title":3},"31":{"body":13,"breadcrumbs":9,"title":3},"32":{"body":15,"breadcrumbs":7,"title":1},"33":{"body":78,"breadcrumbs":9,"title":3},"34":{"body":73,"breadcrumbs":11,"title":5},"35":{"body":141,"breadcrumbs":14,"title":8},"36":{"body":15,"breadcrumbs":10,"title":4},"37":{"body":9,"breadcrumbs":9,"title":3},"38":{"body":31,"breadcrumbs":10,"title":4},"39":{"body":133,"breadcrumbs":8,"title":2},"4":{"body":199,"breadcrumbs":6,"title":3},"40":{"body":92,"breadcrumbs":8,"title":2},"41":{"body":42,"breadcrumbs":8,"title":2},"42":{"body":34,"breadcrumbs":11,"title":4},"43":{"body":51,"breadcrumbs":9,"title":2},"44":{"body":51,"breadcrumbs":15,"title":8},"45":{"body":50,"breadcrumbs":12,"title":5},"46":{"body":40,"breadcrumbs":8,"title":3},"47":{"body":21,"breadcrumbs":7,"title":2},"48":{"body":106,"breadcrumbs":7,"title":2},"49":{"body":0,"breadcrumbs":3,"title":1},"5":{"body":73,"breadcrumbs":5,"title":2},"50":{"body":0,"breadcrumbs":6,"title":2},"51":{"body":0,"breadcrumbs":8,"title":3},"52":{"body":23,"breadcrumbs":6,"title":2},"53":{"body":37,"breadcrumbs":6,"title":2},"54":{"body":33,"breadcrumbs":7,"title":3},"55":{"body":51,"breadcrumbs":7,"title":3},"56":{"body":58,"breadcrumbs":11,"title":7},"57":{"body":302,"breadcrumbs":3,"title":2},"6":{"body":203,"breadcrumbs":5,"title":2},"7":{"body":10,"breadcrumbs":3,"title":2},"8":{"body":208,"breadcrumbs":6,"title":4},"9":{"body":317,"breadcrumbs":6,"title":4}},"docs":{"0":{"body":"Welcome to the developer documentation for Walrus, a decentralized storage and availability protocol designed specifically for large binary files, or \"blobs\". Walrus focuses on providing a robust solution for storing unstructured content on decentralized storage nodes while ensuring high availability and reliability even in the presence of Byzantine faults.","breadcrumbs":"Walrus » Walrus","id":"0","title":"Walrus"},"1":{"body":"Storage and retrieval : Walrus supports storage operations to write and read blobs. It also allows anyone to prove that a blob has been stored and is available for retrieval at a later time. Cost efficiency : By utilizing advanced error correction coding, Walrus maintains storage costs at approximately five times the size of the stored blobs and encoded parts of each blob are stored on each storage node. This is significantly more cost-effective compared to traditional full replication methods and much more robust against failures compared to protocols that only store each blob on a subset of storage nodes. Integration with Sui blockchain : Walrus leverages the Sui for coordination, attesting availability and payments. Storage space can be owned as a resource on Sui, split, merged, and transferred. Blob storage is represented using storage objects on Sui, and smart contracts can check whether a blob is available and for how long. Flexible access : Users can interact with Walrus through a command-line interface (CLI), software development kits (SDKs), and web2 HTTP technologies. Walrus is designed to work well with traditional caches and content distribution networks (CDNs), while ensuring all operations can also be run using local tools to maximize decentralization.","breadcrumbs":"Walrus » Features","id":"1","title":"Features"},"10":{"body":"We summarize here the basic encoding and cryptographic techniques used in Walrus. Storage nodes hold one or many shards in a storage epoch, out of a larger total (say 1000) and each shard contains one blob sliver for each blob past PoA. Each shard is assigned to a storage node in a storage epoch. An erasure code encode algorithm takes a blob, and encodes it as $K$ symbols, such that any fraction $p$ of symbols can be used to reconstruct the blob. Each blob sliver contains a fixed number of such symbols. We select $p<1/3$ so that a third of symbols and also slivers may be used to reconstruct the blob by the decode algorithm . The matrix used to produce the erasure code is fixed and the same for all blobs by the Walrus system, and encoders have no discretion about it. Storage nodes manage one or more shards, and corresponding sliver of each blob are distributed to all the storage shards. As a result, the overhead of the distributed store is ~5x that of the blob itself, no matter how many shards we have. The encoding is systematic meaning that some storage nodes hold part of the plain blob, allowing for fast random access reads. Each blob is also associated with some metadata including a blob ID to allow verification: A blob ID is computed as an authenticator of the set of all shard data and metadata (byte size, encoding, blob hash). We hash a sliver representation in each of the shards and add the resulting hashes into a Merkle tree. Then the root of the Merkle tree is the blob hash used to derive the blob ID that identifies the blob in the system. Each storage node may use the blob ID to check if some shard data belongs to a blob using the authenticated structure corresponding to the blob hash (Merkle tree). A successful check means that the data is indeed as intended by the writer of the blob (who, remember, may be corrupt). When any party reconstructs a blob ID from shards data and slivers, or accepts any blob purporting to be a specific blob ID, it must check that it encodes to the correct blob ID. This process involves re-coding the blob using the erasure correction code, and re-deriving the blob ID to check the blob indeed matches it. This prevents a malformed blob (i.e., incorrectly erasure coded) from ever being read with a blob ID at any correct recipient. A set of slivers above the reconstruction threshold belonging to a blob ID that are either inconsistent or lead to the reconstruction of a different ID represent an incorrect encoding (this may happen if the user that encoded the blob was malicious and encoded it incorrectly). Storage nodes may delete slivers belonging to inconsistently encoded blobs, and upon request return an inconsistency proof.","breadcrumbs":"Overview » Encoding » Encoding, overheads, and verification","id":"10","title":"Encoding, overheads, and verification"},"11":{"body":"Walrus operations can be separated in interactions with the Sui chain , which is used by Walrus for coordination and governance, and off-chain interactions between clients and storage nodes.","breadcrumbs":"Operations » Operations","id":"11","title":"Operations"},"12":{"body":"Walrus uses Sui smart contracts to coordinate storage operations as resources that have a lifetime, and payments. As well as to facilitate governance to determine the storage nodes holding each storage shard. We outline here these operations and refer to them below as part of the read / write paths. As a reminder, only blob metadata is ever exposed to Sui or its validators, and the content of blobs is always stored off-chain on Walrus storage nodes and caches. The storage nodes or caches do not have to overlap with any Sui infra (validators etc), and the storage epochs may be of different lengths and not have the same start / end times as Sui epochs.","breadcrumbs":"Operations » Sui operations » Operations on Sui","id":"12","title":"Operations on Sui"},"13":{"body":"A number of Sui smart contracts hold the metadata of the Walrus system and all its entities. A Walrus system object holds the committee of storage nodes for the current storage epoch. The system object also holds the total available space on Walrus and the price per unit of storage (1 KiB). These values are determined by 2/3 agreement between the storage nodes for the storage epoch. Users can pay to purchase storage space for some time duration. These space resources may be split, merged and transferred. Later they can be used to place a blob ID into Walrus. The storage fund holds funds for storing blobs over one, multiple storage epochs or perpetually. When purchasing storage space from the system object users pay into the storage fund separated over multiple storage epochs, and payments are made each epoch to storage nodes according to performance (see below). A user acquires some storage through the contracts or transfer, and can assign to it a blob ID, signifying they wish to store this blob ID into it. This emits a Move resource event that both caches and storage nodes listen to to expect and authorize off-chain storage operations. Eventually a user holds an off-chain availability certificate from storage nodes for a blob ID. The user uploads the certificate on chain to signal that the blob ID is available for an availability period. The certificate is checked against the latest Walrus committee, and an availability event is emitted for the blob ID if correct. This is the PoA for the blob. In case a blob ID is not correctly encoded a inconsistency proof certificate may be uploaded on chain at a later time, and an inconsistent blob event is emitted signaling to all that the blob ID read results will always return None. This indicates that its slivers may be deleted by storage nodes, except for an indicator to return None. Users writing to Walrus, need to perform Sui transactions to acquire storage and certify blobs. Users creating or consuming proofs for attestations of blob availability read the chain only to prove or verify emission of events. A node that reads Walrus resources only needs to read the blockchain to get committee metadata once per epoch, and then they request slivers directly from storage nodes by blob ID to perform reads.","breadcrumbs":"Operations » Sui operations » Storage resource life cycle on Sui","id":"13","title":"Storage resource life cycle on Sui"},"14":{"body":"Each Walrus storage epoch is represented by the Walrus system object that contains a storage committee and various metadata or storage nodes like the mapping between shards and storage nodes, available space and current costs. User may go to the system object for the period and buy some storage amount for one or more storage epochs. At each storage epoch there is a price for storage, and the payment provided becomes part of a storage fund for all the storage epochs that span the storage bought. There is a maximum number of storage epochs in the future for which storage can be bought (~2 years). Storage is a resource that can be split and merged, and transferred. At the end of the storage epoch part of the funds in the storage fund need to be allocated to storage nodes . The idea here is for storage nodes to perform light audits of each others, and suggest which nodes are to be paid based on the performance of these audits.","breadcrumbs":"Operations » Sui operations » Governance operations on Sui","id":"14","title":"Governance operations on Sui"},"15":{"body":"Walrus operations happen off Sui, but may interact with the Sui flows defining the resource life cycle.","breadcrumbs":"Operations » Off-chain operations » Off-chain operations","id":"15","title":"Off-chain operations"},"16":{"body":"Write paths of Walrus Systems overview of writes, illustrated above: A user acquires a storage resource of appropriate size and duration on-chain, either by directly buying it on the Walrus system object, or a secondary market. A user can split, merge, and transfer storage acquired storage resources. When a user wants to write a blob, it first erasure codes it using encode, and computes its blob ID. Then they can perform the following steps itself, or use a publisher to perform steps on their behalf. The user goes on chain (Sui) and updates a storage resource to register the blob ID with the appropriate size and lifetime desired. This emits an event, received by storage nodes. Once the user receives it then continues the upload. The user sends each of the blob slivers and metadata to the storage nodes that currently manages the corresponding shards. A storage node managing a shard receives a sliver and checks it against the blob ID of the overall blob. It also checks that there is a blob resource with that blob ID that is authorized to store a blob. If correct, then it signs a statement that it holds the sliver for blob ID (and metadata) and returns it to the user. The user puts together the signatures returned from storage nodes into an availability certificate and sends it on chain. When successfully checked an availability event for the blob ID is emitted, and all other storage nodes seek to download any missing shards for the blob ID. This event being emitted on Sui is the Point of Availability (PoA) for the blob ID. After the PoA, and without user involvement, storage nodes sync and recover any missing slivers that are certified. The user waits for 2/3 of shards signatures to return a certificate of availability. The rate of the code is below 1/3 allowing for reconstruction if even 1/3 of shards only return the sliver. Since at most 1/3 of the storage nodes can fail, this ensures reconstruction if a reader requests slivers from all storage nodes that have signed the ID of the blob. Note that the full process can be mediated by a publisher, that receives a blob and drives the process to completion.","breadcrumbs":"Operations » Off-chain operations » Write paths","id":"16","title":"Write paths"},"17":{"body":"Since no content data is required to refresh the period of storage, refresh is conducted fully on chain within the protocol. To request an extension to the availability period of a blob, a user provides an appropriate storage resource. Upon success this emits an event that storage nodes receive to extend the period each sliver is stored for.","breadcrumbs":"Operations » Off-chain operations » Refresh availability","id":"17","title":"Refresh availability"},"18":{"body":"When a correct storage node tries to reconstruct a shard it may fail if the encoding of a blob ID past PoA was incorrect, and will be able to extract an inconsistency proof for the blob ID. It then generates a inconsistency certificate and uploads it on chain. The flow is as follows: A storage node fails to reconstruct a shard, and generates an inconsistency proof. The storage node sends the blob ID and inconsistency proof to all storage nodes of the storage epoch, and gets a signature, that it aggregates to an inconsistency certificate. The storage node sends the inconsistency certificate to the Walrus smart contract, that checks it and emits a inconsistent resource event. Upon receiving a inconsistent resource event correct storage nodes delete sliver data and only keep a metadata record to return None for the blob ID for the availability period. No storage attestation challenges are issued for this blob ID. Note that a blob ID that is inconsistent will always resolve to None upon reading: this is due to the read process running the decoding algorithm, and then re-encoding to check the blob ID is correctly derived from a consistent encoding. This means that an inconsistency proof only reveals a true fact to storage nodes (that may not otherwise have ran decoding), and does not change the output of read in any case. Note however that partial reads leveraging the systematic nature of the encoding may return partial reads for inconsistently encoded files. Thus if consistency and availability of reads is important dapps should do full reads rather than partial reads.","breadcrumbs":"Operations » Off-chain operations » Inconsistent resource flow","id":"18","title":"Inconsistent resource flow"},"19":{"body":"A user can read stored blobs either directly or through a cache. We discuss here the direct user journey since this is also the operation of the cache in case of a cache miss. We assume that most reads will happen through caches, for blobs that are hot, and will not result in requests to storage nodes. The reader gets the metadata for the blob ID from any storage node, and authenticates it using the blob ID. The reader then sends a request for the shards corresponding to blob ID to storage nodes, and waits for f+1 to respond. Sufficient requests are sent in parallel to ensure low latency for reads. The reader authenticates the slivers returned with the blob ID, reconstructs the blob, and decides whether the contents are a valid blob or inconsistent. Optionally, for a cache, the result is cached and can be served without re-construction for some time, until it is removed from the cache. Requests for the blob to the cache return the blob contents, or a proof the blob is inconsistently encoded.","breadcrumbs":"Operations » Off-chain operations » Read paths","id":"19","title":"Read paths"},"2":{"body":"Walrus's architecture ensures that content remains accessible and retrievable even when many storage nodes are unavailable or malicious. Under the hood it uses modern error correction techniques based on fast linear fountain codes, augmented to ensure resilience against Byzantine faults, and a dynamically changing set of storage nodes. The core of Walrus remains simple, and storage node management and blob certification leverages Sui smart contracts. This documentation is split into several parts. The first part provides an overview of the objectives, security properties, and architecture of the Walrus system. The second part contains concrete documentation on the usage of Walrus. At the end, we provide a glossary , which defines key terms used throughout the project. Walrus is architected to provide a reliable and cost-effective solution for large-scale blob storage, making it an ideal choice for applications requiring decentralized, affordable, durable, and accessible data storage.","breadcrumbs":"Walrus » Architecture and operations","id":"2","title":"Architecture and operations"},"20":{"body":"During an epoch a correct storage node challenges all shards to provide blob slivers past PoA: The list of available blobs for the period is determined by the sequence of Sui events up to the past period. Inconsistent blobs are not challenged, and a record proving this status can be returned instead. A challenge sequence is determined by providing a seed to the challenged shard. The sequence is then computed based both on the seed AND the content of each challenged blob ID. This creates a sequential read dependency. The response to the challenge provides the sequence of shard contents for the blob IDs in a timely manner. The challenger node uses thresholds to determine whether the challenge was passed, and reports the result on chain. The challenge / response communication is authenticated. Challenges provide some reassurance that the storage node actually can recover shard data in a probabilistic manner, avoiding storage nodes getting payment without any evidence they may retrieve shard data. The sequential nature of the challenge and some reasonable timeout also ensure that the process is timely.","breadcrumbs":"Operations » Off-chain operations » Challenge mechanism for storage attestation","id":"20","title":"Challenge mechanism for storage attestation"},"21":{"body":"In this document, we left out details of the following features: Shard transfer and recovery upon storage epoch change. The encoding scheme used has been designed to allow this operation to be efficient. A storage node needs to only get data of the same magnitude to the missing sliver data to reconstruct them. Details of light clients that can be used to sample availability. Individual clients may sample the certified blobs from Sui metadata, and sample the availability of some slivers that they store. On-chain bounties may be used to retrieve these slivers for missing blobs.","breadcrumbs":"Future discussion » Future discussion","id":"21","title":"Future discussion"},"22":{"body":"We provide 3 ways to interact directly with the Walrus storage system: Through the walrus client command line interface (CLI) . Through a JSON API of the walrus CLI. Through an HTTP API exposed by the walrus client daemon.","breadcrumbs":"Interacting with Walrus » Interacting with Walrus","id":"22","title":"Interacting with Walrus"},"23":{"body":"The walrus binary can be used to interact with Walrus as a client. To use it, you need a Walrus configuration and a Sui wallet. Detailed usage information is available through walrus --help Storing and reading blobs from Walrus can be achieved through the following commands: CONFIG=working_dir/client_config.yaml # adjust for your configuration file\nwalrus -c $CONFIG store0x6412c4cfbe50e219c2d4d30108d7321d064e15bf64e752307100bff5eb91da38
.
The steps that follow require that the registration object is owned by the address you are using on -the Sui CLI.
-To find the address, execute:
+The steps that follow require that the SuiNS registration object is owned by the address you are +using on the Sui CLI. Therefore, we need to send this registration object from the address you use +in your browser wallet, to the address of your Sui CLI.
+To find the Sui CLI address, execute:
sui client active-address
-Then, from your browser wallet, select "Assets", look for the NFT of the SuiNS registration, which -should look as follows:
+Then, from your browser wallet, select the "Assets" tab, and look for the NFT of the SuiNS +registration, which should look as follows:
Click on it, scroll down to "Send NFT", and send it to the address discovered with the command above. Now, your Sui CLI address owns the registration NFT, and you can proceed to the next step.