From 911891b7411aa04c6252a5c1a8cd9003835df1ff Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 24 Sep 2024 18:13:03 +0530 Subject: [PATCH 1/4] fix consensus done in upload --- zboxcore/sdk/chunked_upload_model.go | 2 +- zboxcore/sdk/chunked_upload_process_js.go | 24 ++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/zboxcore/sdk/chunked_upload_model.go b/zboxcore/sdk/chunked_upload_model.go index 356272ee1..0eb5b9724 100644 --- a/zboxcore/sdk/chunked_upload_model.go +++ b/zboxcore/sdk/chunked_upload_model.go @@ -91,7 +91,7 @@ type ChunkedUpload struct { //used in wasm check chunked_upload_process_js.go listenChan chan struct{} //nolint:unused //used in wasm check chunked_upload_process_js.go - processMap map[int]int //nolint:unused + processMap map[int]zboxutil.Uint128 //nolint:unused //used in wasm check chunked_upload_process_js.go processMapLock sync.Mutex //nolint:unused } diff --git a/zboxcore/sdk/chunked_upload_process_js.go b/zboxcore/sdk/chunked_upload_process_js.go index 6c4022056..de135b572 100644 --- a/zboxcore/sdk/chunked_upload_process_js.go +++ b/zboxcore/sdk/chunked_upload_process_js.go @@ -151,6 +151,7 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, var thumbnailChunkData []byte worker := jsbridge.GetWorker(blobber.blobber.ID) if worker == nil { + logger.Logger.Error("worker not found for blobber: ", blobber.blobber.Baseurl) continue } if len(thumbnailShards) > 0 { @@ -192,6 +193,9 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, err = worker.PostMessage(safejs.Safe(obj), []safejs.Value{safejs.Safe(fileshardUint8.Get("buffer"))}) if err == nil { successCount++ + } else { + logger.Logger.Error("error posting message to worker: ", err) + su.uploadMask = su.uploadMask.And(zboxutil.NewUint128(1).Lsh(pos).Not()) } if isFinal { blobber.fileRef.ChunkSize = su.chunkSize @@ -300,7 +304,7 @@ func (su *ChunkedUpload) listen(allEventChan []eventChanWorker, respChan chan er //get error message //get final result var err error - isFinal, err = su.processWebWorkerUpload(data, blobber) + isFinal, err = su.processWebWorkerUpload(data, blobber, pos) if err != nil { errC := atomic.AddInt32(&errCount, 1) if errC >= int32(su.consensus.consensusThresh) { @@ -308,13 +312,11 @@ func (su *ChunkedUpload) listen(allEventChan []eventChanWorker, respChan chan er } } else { uploadSuccess = true - su.consensus.Done() } return default: logger.Logger.Error("unknown msg type: ", msgType) } - return } }(pos) @@ -335,9 +337,9 @@ func (su *ChunkedUpload) listen(allEventChan []eventChanWorker, respChan chan er su.ctxCncl(err) respChan <- err } - for chunkEndIndex, count := range su.processMap { - if count >= su.consensus.consensusThresh { - su.updateProgress(chunkEndIndex, su.uploadMask) + for chunkEndIndex, mask := range su.processMap { + if mask.CountOnes() >= su.consensus.consensusThresh { + su.updateProgress(chunkEndIndex, mask) delete(su.processMap, chunkEndIndex) } } @@ -349,7 +351,7 @@ func (su *ChunkedUpload) listen(allEventChan []eventChanWorker, respChan chan er } } -func (su *ChunkedUpload) processWebWorkerUpload(data *safejs.Value, blobber *ChunkedUploadBlobber) (bool, error) { +func (su *ChunkedUpload) processWebWorkerUpload(data *safejs.Value, blobber *ChunkedUploadBlobber, pos uint64) (bool, error) { var isFinal bool success, err := data.Get("success") if err != nil { @@ -368,7 +370,7 @@ func (su *ChunkedUpload) processWebWorkerUpload(data *safejs.Value, blobber *Chu chunkEndIndexObj, _ := data.Get("chunkEndIndex") chunkEndIndex, _ := chunkEndIndexObj.Int() - su.updateChunkProgress(chunkEndIndex) + su.updateChunkProgress(chunkEndIndex, pos) finalRequestObject, _ := data.Get("isFinal") finalRequest, _ := finalRequestObject.Bool() if finalRequest { @@ -693,7 +695,7 @@ type eventChanWorker struct { func (su *ChunkedUpload) startProcessor() { su.listenChan = make(chan struct{}, su.uploadWorkers) - su.processMap = make(map[int]int) + su.processMap = make(map[int]zboxutil.Uint128) su.uploadWG.Add(1) go func() { respChan := make(chan error, 1) @@ -734,8 +736,8 @@ func (su *ChunkedUpload) startProcessor() { }() } -func (su *ChunkedUpload) updateChunkProgress(chunkEndIndex int) { +func (su *ChunkedUpload) updateChunkProgress(chunkEndIndex int, pos uint64) { su.processMapLock.Lock() - su.processMap[chunkEndIndex] += 1 + su.processMap[chunkEndIndex] = su.processMap[chunkEndIndex].Or(zboxutil.NewUint128(1).Lsh(pos)) su.processMapLock.Unlock() } From dc21d36e736a30c36f9867b3f8c1520d4740e960 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 24 Sep 2024 18:15:49 +0530 Subject: [PATCH 2/4] revert num blocks to 60 --- wasmsdk/blobber.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index ebe163b94..3f4add853 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -778,11 +778,10 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) { RemotePath: fullRemotePath, CustomMeta: option.CustomMeta, } - // numBlocks := option.NumBlocks - // if numBlocks <= 1 { - // numBlocks = 100 - // } - numBlocks := 60 + numBlocks := option.NumBlocks + if numBlocks <= 1 { + numBlocks = 100 + } options := []sdk.ChunkedUploadOption{ sdk.WithThumbnail(option.ThumbnailBytes.Buffer), From 2659df7175cd2a85588f656523c5929db726d3fb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 26 Sep 2024 22:50:00 +0530 Subject: [PATCH 3/4] revert readme changes --- README.md | 238 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 220 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index bc6f9bb43..369499924 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,235 @@ - +# 0chain/gosdk -# gosdk +The Züs client SDK written in Go programming language. -```go -import "github.com/0chain/gosdk" +- [GoSDK - a Go based SDK for Züs dStorage]() + - [Züs Overview ](#overview) + - [Installation](#installation) + - [Mobile Builds(iOS and Android)](#mobile-builds) + - [Exposing a gosdk function to mobilesdk](#expose-a-gosdk-function-to-mobilesdk) + - [Export a gosdk function to WebAssembly](#export-a-gosdk-function-to-webassembly) + - [Running Unit Tests](#running-unit-tests) + - [FAQ](#faq) + +## Overview + +[Züs](https://zus.network/) is a high-performance cloud on a fast blockchain offering privacy and configurable uptime. It is an alternative to traditional cloud S3 and has shown better performance on a test network due to its parallel data architecture. The technology uses erasure code to distribute the data between data and parity servers. Züs storage is configurable to provide flexibility for IT managers to design for desired security and uptime, and can design a hybrid or a multi-cloud architecture with a few clicks using [Blimp's](https://blimp.software/) workflow, and can change redundancy and providers on the fly. + +For instance, the user can start with 10 data and 5 parity providers and select where they are located globally, and later decide to add a provider on-the-fly to increase resilience, performance, or switch to a lower cost provider. + +Users can also add their own servers to the network to operate in a hybrid cloud architecture. Such flexibility allows the user to improve their regulatory, content distribution, and security requirements with a true multi-cloud architecture. Users can also construct a private cloud with all of their own servers rented across the globe to have a better content distribution, highly available network, higher performance, and lower cost. + +[The QoS protocol](https://medium.com/0chain/qos-protocol-weekly-debrief-april-12-2023-44524924381f) is time-based where the blockchain challenges a provider on a file that the provider must respond within a certain time based on its size to pass. This forces the provider to have a good server and data center performance to earn rewards and income. + +The [privacy protocol](https://zus.network/build) from Züs is unique where a user can easily share their encrypted data with their business partners, friends, and family through a proxy key sharing protocol, where the key is given to the providers, and they re-encrypt the data using the proxy key so that only the recipient can decrypt it with their private key. + +Züs has ecosystem apps to encourage traditional storage consumption such as [Blimp](https://blimp.software/), a S3 server and cloud migration platform, and [Vult](https://vult.network/), a personal cloud app to store encrypted data and share privately with friends and family, and [Chalk](https://chalk.software/), a high-performance story-telling storage solution for NFT artists. + +Other apps are [Bolt](https://bolt.holdings/), a wallet that is very secure with air-gapped 2FA split-key protocol to prevent hacks from compromising your digital assets, and it enables you to stake and earn from the storage providers; [Atlus](https://atlus.cloud/), a blockchain explorer and [Chimney](https://demo.chimney.software/), which allows anyone to join the network and earn using their server or by just renting one, with no prior knowledge required. + +## Installation + +### Supported Platforms + +This repository currently supports the following platforms: + +- Mac OSX Mojave 10.14.5 or Above +- Linux (Ubuntu/bionic): This includes all Ubuntu 18+ platforms, so Ubuntu 19, Linux Mint 19 etc. (apt based package installer) +- Linux (RHEL/CENTOS 7+): All Releases based on RHEL 7+, Centos 7+, Fedora 30 etc. (yum based package installer) + +### Instructions + +- Go is required to build gosdk code. Instructions can be found [here](https://go.dev/doc/install) + +1. Save below code as `sdkversion.go` + + package main + + import ( + "fmt" + + "github.com/0chain/gosdk/zcncore" + ) + + func main() { + fmt.Println("gosdk version: ", zcncore.GetVersion()) + } + +2. Run below command: (if you don't have gosdk already in your GOPATH) + + go get github.com/0chain/gosdk + +3. Build the sample application sdkversion + + go build -o sdkversion sdkversion.go + +4. Run the executable + + ./sdkversion + +5. If it prints the gosdk version installed then setup is complete. + +## Mobile Builds + +- gosdk can be built for iOS and Android using gomobile. +- Xcode Command Line Tools is required to build the SDK for iOS. +- Android studio with NDK is required to build the SDK for Android. +- See [FAQ](#faq) for installing Go, gomobile Xcode or Android Studio. + +Steps: + +1. Run the command below for the first time to setup the gomobile environment: + + make setup-gomobile + +2. In case the Go package is not found in `golang.org/x/mobile/bind`, run: + `go get golang.org/x/mobile/bind` +3. Run below commands in the root folder of the repo to build the Mobile SDK: + + For iOS only: + make build-ios + For Android only: + make build-android + +## Expose a gosdk function to mobilesdk + +Examples: + +- `mobilesdk/sdk/common.go`, which exports the functions in `core/encryption/hash.go`. + +Steps: + +1. If you are exposing: + + - a new function from an existing file, such as `zboxcore/sdk/allocation.go`, you should add a function to `mobilesdksdk/zbox/allocation.go`. This new function should call the gosdk function you intend to expose. + - a function from a new file, you should create a new `.go` file for it. This should follow the same style as `mobilesdksdk/zbox/allocation.go`. In the new file, call the gosdk function you intend to expose. + +2. Build the Mobile SDK as mentioned in the 'Mobile Builds' section of this file to build the aar file used in the mobile application you are developing. + +## Export a gosdk function to WebAssembly + +Examples: + +- `wasmsdk/ethwallet.go` which exports the functions in `zcncore/ethwallet.go`. +- `wasmsdk/wallet.go` which exports one function in `zcncore/wallet.go`. + +Steps: + +1. If you are exporting: + + - a new function from `zcncore/wallet.go`, you should add to `wasmsdk/wallet.go` + + - a function from a new file, you should create a new `.go` file for it, in the same style as `wasmsdk/wallet.go` or `wasmsdk/ethwallet.go` + +2. In func main(), `https://github.com/0chain/gosdk/wasmsdk/proxy.go`, you need to add this line: + + ```golang + js.Global().Set("YOURFUNC", js.FuncOf(YOURFUNC)) + ``` + +3. Now you need to compile a new `.wasm` (e.g. proxy.wasm). Currently, the right version to compile wasm is with Go version 1.16. So make sure you have it to make the wasm build works properly. In order to compile, run the following command: + + ```bash + $ GOOS=js CGO_ENABLED=0 GOARCH=wasm go build -o .wasm github.com/0chain/gosdk/wasmsdk + ``` + +### An important note regarding export of an async function + +If your golang function needs to suport asynchronous execution, you need to add more wrapper code where you are returning a Promise object. + +See "InitZCNSDK" example: + +```golang +func InitZCNSDK(this js.Value, p []js.Value) interface{} { + blockWorker := p[0].String() + signscheme := p[1].String() + + handler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + resolve := args[0] + // reject := args[1] + + go func() { + err := zcncore.InitZCNSDK(blockWorker, signscheme) + if err != nil { + fmt.Println("error:", err) + } + resolve.Invoke() + }() + + return nil + }) + + // Create and return the Promise object + promiseConstructor := js.Global().Get("Promise") + return promiseConstructor.New(handler) +} ``` -## Index +## Running Unit Tests + +### BLS unit test + +It's advisable to put GOPATH as `$TOP/../go`, to avoid conflicts with this command: `go build ./...` + +To run all the unit tests in `gosdk`: `go test github.com/0chain/gosdk/zboxcore/sdk -v` -- [type Timestamp](<#Timestamp>) - - [func Now\(\) Timestamp](<#Now>) +```bash +$ go test ./... +``` +To run all the unit tests in `bls0chain_test.go`, run this command from $TOP: `go test github.com/0chain/gosdk/core/zcncrypto -v` - -## type [Timestamp]() +To run a specific unit test in `bls0chain_test.go`, such as `TestSignatureScheme`, run: `go test github.com/0chain/gosdk/core/zcncrypto -v -run TestSignatureScheme` -Timestamp represents Unix time \(e.g. in seconds\) +To run the coverage test in `gosdk`: -```go -type Timestamp int64 +```bash +$ go test -coverprofile=coverage.out +$ go tool cover -html=coverage.out ``` - -### func [Now]() +### WebAssembly + +#### Using go test + +1. You need to install nodejs first, see [this page](https://nodejs.org/en/download/) for further instructions + +2. Add `/path/to/go/misc/wasm` to your `$PATH` environment variable (so that `go test` can find `go_js_wasm_exec`). For example in Ubuntu, run `$export PATH=$PATH:/usr/local/go/misc/wasm/`. + +3. You can then run the test by following the [BLS unit test](#bls-unit-test) above by adding the prefix environment `GOOS=js CGO_ENABLED=0 GOARCH=wasm`: + ```bash + go test -tags test -v github.com/0chain/gosdk/wasmsdk + ``` + +#### Test in the client + +1. After you successfully [export the wasm package to proxy.wasm](#how-to-export-a-gosdk-function-to-webassembly), you can test the exported `proxy.wasm`. + +2. We currently have a test page going at the js-client-sdk repo: `https://github.com/0chain/js-client-sdk/blob/gosdk/test/index.html` + +3. You can replace the proxy.wasm at `https://github.com/0chain/js-client-sdk/blob/gosdk/test/proxy.wasm` + +4. You need to start a special test server in order to stream WASM files. Use the following command from js-client-sdk $TOP: `sudo php -S localhost:82 test/server.php` + +5. See "testethwallet" function in index.html for how the testing for ethwallet.go is done. + +6. To test the function you exported, it's probably as simple as calling "HelloWorld()". It should be a 1-liner. + +### How to install `ffmpeg` + +#### On Ubuntu Linux -```go -func Now() Timestamp +```bash +sudo apt-get install ffmpeg +sudo apt-get install v4l-utils ``` -Now \- current datetime +### FAQ -Generated by [gomarkdoc]() +- [How to install GO on any platform](https://golang.org/doc/install) +- [How to install different version of GO](https://golang.org/doc/install#extra_versions) +- [How to use go mod](https://blog.golang.org/using-go-modules) +- [What is gomobile](https://godoc.org/golang.org/x/mobile/cmd/gomobile) +- [About XCode](https://developer.apple.com/xcode/) +- [Android Studio](https://developer.android.com/studio) +- [Android NDK](https://developer.android.com/ndk/) From 58af8eb4bb1c1b9493da160a475f6055d08529a8 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 7 Oct 2024 18:40:08 +0530 Subject: [PATCH 4/4] add retries and update fasthttp --- zboxcore/sdk/blockdownloadworker.go | 2 +- zboxcore/sdk/chunked_upload_process_js.go | 12 ++++++++++++ zboxcore/sdk/rollback.go | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/zboxcore/sdk/blockdownloadworker.go b/zboxcore/sdk/blockdownloadworker.go index 603003903..eb62d1df2 100644 --- a/zboxcore/sdk/blockdownloadworker.go +++ b/zboxcore/sdk/blockdownloadworker.go @@ -175,7 +175,7 @@ func (req *BlockDownloadRequest) downloadBlobberBlock(fastClient *fasthttp.Clien return errors.New(RateLimitError, "Rate limit error") } - if statuscode == http.StatusInternalServerError { + if statuscode == http.StatusInternalServerError || statuscode == http.StatusBadGateway { shouldRetry = true return errors.New("internal_server_error", "Internal server error") } diff --git a/zboxcore/sdk/chunked_upload_process_js.go b/zboxcore/sdk/chunked_upload_process_js.go index 6c4022056..a4a7f0eb8 100644 --- a/zboxcore/sdk/chunked_upload_process_js.go +++ b/zboxcore/sdk/chunked_upload_process_js.go @@ -629,8 +629,12 @@ func sendUploadRequest(dataBuffers []*bytes.Buffer, contentSlice []string, blobb if err != nil { logger.Logger.Error("Upload : ", err) if errors.Is(err, fasthttp.ErrConnectionClosed) || errors.Is(err, syscall.EPIPE) || errors.Is(err, fasthttp.ErrDialTimeout) { + err = ErrNetwork return err, true } + if errors.Is(err, fasthttp.ErrTimeout) { + return ErrNetwork, false + } return fmt.Errorf("Error while doing reqeust. Error %s", err), false } @@ -652,6 +656,14 @@ func sendUploadRequest(dataBuffers []*bytes.Buffer, contentSlice []string, blobb return } + if resp.StatusCode() == http.StatusBadGateway { + logger.Logger.Error("Got bad gateway error") + time.Sleep(1 * time.Second) + shouldContinue = true + err = ErrNetwork + return + } + msg := string(respbody) logger.Logger.Error(blobberURL, " Upload error response: ", resp.StatusCode(), diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index 3a2a8485d..d15a5df1d 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -46,6 +46,7 @@ const ( var ( ErrRetryOperation = errors.New("retry_operation") ErrRepairRequired = errors.New("repair_required") + ErrNetwork = errors.New("network_error") ) type RollbackBlobber struct {