Skip to content

Commit

Permalink
Drop HTTP, update REST (#75)
Browse files Browse the repository at this point in the history
  • Loading branch information
roman-khimov authored Mar 22, 2024
2 parents 662efc4 + cbe79f5 commit bb0dacd
Show file tree
Hide file tree
Showing 10 changed files with 27 additions and 105 deletions.
3 changes: 1 addition & 2 deletions .env
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
NEOGO_VERSION=0.105.1
AIO_VERSION=0.40.1
HTTPGW_VERSION=0.28.0
RESTGW_VERSION=0.7.1
RESTGW_VERSION=0.8.2
1 change: 0 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,5 @@ jobs:
NEOGO_HUB_TAG=${{ steps.setver.outputs.NEOGO_VERSION }}
NEOFS_HUB_IMAGE=nspccdev/neofs
NEOFS_HUB_TAG=${{ steps.setver.outputs.AIO_VERSION }}
NEOFS_HTTP_HUB_TAG=${{ steps.setver.outputs.HTTPGW_VERSION }}
NEOFS_REST_HUB_TAG=${{ steps.setver.outputs.RESTGW_VERSION }}
tags: nspccdev/neofs-aio:${{ steps.setver.outputs.AIO_VERSION }}${{ steps.setlatest.outputs.latest }}
7 changes: 1 addition & 6 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@ ARG NEOFS_HUB_IMAGE=nspccdev/neofs
ARG NEOFS_HUB_TAG=latest
ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
ARG NEOGO_HUB_TAG=latest
ARG NEOFS_HTTP_HUB_TAG=latest
ARG NEOFS_REST_HUB_TAG=latest

FROM ${NEOGO_HUB_IMAGE}:${NEOGO_HUB_TAG} as neo-go
FROM ${NEOFS_HUB_IMAGE}-cli:${NEOFS_HUB_TAG} as neofs-cli
FROM ${NEOFS_HUB_IMAGE}-ir:${NEOFS_HUB_TAG} as neofs-ir
FROM ${NEOFS_HUB_IMAGE}-storage:${NEOFS_HUB_TAG} as neofs-storage
FROM ${NEOFS_HUB_IMAGE}-adm:${NEOFS_HUB_TAG} as neofs-adm
FROM ${NEOFS_HUB_IMAGE}-http-gw:${NEOFS_HTTP_HUB_TAG} as neofs-http-gw
FROM ${NEOFS_HUB_IMAGE}-rest-gw:${NEOFS_REST_HUB_TAG} as neofs-rest-gw

# Executable image
Expand All @@ -31,22 +29,19 @@ COPY --from=neofs-cli /bin/neofs-cli /usr/bin/neofs-cli
COPY --from=neofs-ir /bin/neofs-ir /usr/bin/neofs-ir
COPY --from=neofs-storage /bin/neofs-node /usr/bin/neofs-node
COPY --from=neofs-adm /bin/neofs-adm /usr/bin/neofs-adm
COPY --from=neofs-http-gw /bin/neofs-http-gw /usr/bin/neofs-http-gw
COPY --from=neofs-rest-gw /bin/neofs-rest-gw /usr/bin/neofs-rest-gw

COPY ./sn/cli-cfg.yaml /config/cli-cfg-sn.yaml
COPY ./sn/wallet.json /config/wallet-sn.json
COPY ./sn/config.yaml /config/config-sn.yaml
COPY ./http/wallet.json /config/wallet-http.json
COPY ./rest-gw/wallet.json /config/wallet-rest.json
COPY ./ir/cli-cfg.yaml /config/cli-cfg-ir.yaml
COPY ./ir/config.yaml /config/config-ir.yaml
COPY ./ir/node-wallet.json /config/node-wallet.json
COPY ./ir/node-config.yaml /config/node-config.yaml
COPY ./bin/ /config/bin

COPY ./http/http.env /config/http.env
COPY ./rest-gw/rest.env /config/rest.env
RUN sed -ri 's,^([^=]+)=(.*)+$,\1=${\1-\2},' /config/http.env /config/rest.env
RUN sed -ri 's,^([^=]+)=(.*)+$,\1=${\1-\2},' /config/rest.env

ENTRYPOINT ["/config/bin/init-aio.sh"]
2 changes: 0 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ image-aio:
--build-arg NEOFS_HUB_TAG=$(AIO_VERSION) \
--build-arg NEOGO_HUB_IMAGE=$(NEOGO_HUB_IMAGE) \
--build-arg NEOGO_HUB_TAG=$(NEOGO_VERSION) \
--build-arg NEOFS_HTTP_HUB_TAG=$(HTTPGW_VERSION) \
--build-arg NEOFS_REST_HUB_TAG=$(RESTGW_VERSION) \
-f Dockerfile \
-t $(AIO_IMAGE):$(AIO_VERSION) .
Expand All @@ -35,6 +34,5 @@ version:
@echo $(VERSION)
@echo "neofs-node: $(AIO_VERSION)"
@echo "neo-go: $(NEOGO_VERSION)"
@echo "neofs-http-gw: $(HTTPGW_VERSION)"
@echo "neofs-rest-gw: $(RESTGW_VERSION)"

31 changes: 17 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,12 @@ First, you need to create a container to store your object. Because there is
only one storage node in the All-in-One setup, you can only have one replica of
your data. Let's create a container using `neofs-cli`. Container creation
requires on-chain operations, so it may take up to 5-10 seconds to complete.
Here we use the pre-generated key of the HTTP Gateway for simplicity.
Here we use the pre-generated key of the REST Gateway for simplicity.

Password for wallet is `one`.

``` sh
$ neofs-cli -r localhost:8080 -w http/wallet.json \
$ neofs-cli -r localhost:8080 -w rest-gw/wallet.json \
--address NPFCqWHfi9ixCJRu7DABRbVfXRbkSEr9Vo \
container create \
--policy "REP 1" --basic-acl public-read --await
Expand Down Expand Up @@ -126,7 +126,7 @@ $ curl http://localhost:8090/v1/containers/GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ
## Put an object with neofs-cli

``` sh
$ neofs-cli -r localhost:8080 -w http/wallet.json \
$ neofs-cli -r localhost:8080 -w rest-gw/wallet.json \
--address NPFCqWHfi9ixCJRu7DABRbVfXRbkSEr9Vo \
object put \
--cid GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46 \
Expand All @@ -136,19 +136,21 @@ $ neofs-cli -r localhost:8080 -w http/wallet.json \
CID: GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46
```

## Put an object via http
## Put an object via REST

``` sh
$ curl -F '[email protected];filename=cat.jpg' \
http://localhost:8081/upload/ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo
http://localhost:8090/v1/upload/ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo
{
"object_id": "B4J4L61X6zFcz5fcmZaCJJNZfFFTE6uT4pz7dqP87m6m",
"container_id": "ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo"
}
```

The full description of HTTP API supported by HTTP Gateway can be found in
[neofs-http-gw repository](https://github.com/nspcc-dev/neofs-http-gw).
The full description of REST API supported by NeoFS REST gateway can be found in
its OpenAPI specification ([http://localhost:8090](http://localhost:8090) shows
it), for more info take a look at [neofs-rest-gw repository](https://github.com/nspcc-dev/neofs-rest-gw)
as well.

## Get an object via nginx

Expand All @@ -167,7 +169,7 @@ x-container-id: ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo
Content-Disposition: inline; filename=cat.jpg
```

Having nginx as a reverse proxy behind NeoFS HTTP Gateway allows you to tune the
Having nginx as a reverse proxy for NeoFS REST Gateway allows you to tune the
behaviour according to your application needs. For example, you can set the
rewriting rules to use the pre-configured container for a specific domain to
simplify the URL. Together with `FilePath` attribute in objects it would give
Expand All @@ -179,19 +181,20 @@ For example:
location / {
set $cid ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo;
rewrite '^(/[0-9a-zA-Z\-]{43,44})$' /get/$cid/$1 break;
rewrite '^/$' /get_by_attribute/$cid/FileName/index.html break;
rewrite '^/([^/]*)$' /get_by_attribute/$cid/FileName/$1 break;
rewrite '^(/.*)$' /get_by_attribute/$cid/FilePath/$1 break;
rewrite '^(/v1/.*)$' $1 break;
rewrite '^(/[0-9a-zA-Z\-]{43,44})$' /v1/get/$cid/$1 break;
rewrite '^/$' /v1/get_by_attribute/$cid/FileName/index.html break;
rewrite '^/([^/]*)$' /v1/get_by_attribute/$cid/FileName/$1 break;
rewrite '^(/.*)$' /v1/get_by_attribute/$cid/FilePath/$1 break;
proxy_pass http://localhost:8081;
proxy_pass http://localhost:8083;
```

This allow us to upload objects with `FilePath` attached and get them as if they
were put in a directory structure of the regular web server.

``` sh
curl -F '[email protected];filename=cat.jpg' -H "X-Attribute-FilePath: /pic/cat.jpg" http://localhost:8081/upload/ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo
curl -F '[email protected];filename=cat.jpg' -H "X-Attribute-FilePath: /pic/cat.jpg" http://localhost:8090/v1/upload/ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo
{
"object_id": "4s3T11pktSfSxRfjpJ7BsiuYr2hi7po6nUQ333SPYkWF",
"container_id": "ADsJLhJhLQRGMufFin56PCTtPK1BiSxbg6bDmdgSB1Mo"
Expand Down
20 changes: 2 additions & 18 deletions bin/init-aio.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
## When we start AIO as a separate container or part of a docker-compose. In this case we move background tasks to foreground.
## Otherwise we may use AIO container like a base for own container, thus we need just run all AIO services, before our service.
IS_PURE_START="true"
## if "true" http service will be started
IS_START_HTTP="true"
## if "true" rest service will be started
IS_START_REST="true"

Expand All @@ -13,9 +11,6 @@ while getopts d:h:r: option; do
d)
IS_PURE_START="$OPTARG"
;;
h)
IS_START_HTTP="$OPTARG"
;;
r)
IS_START_REST="$OPTARG"
;;
Expand All @@ -41,9 +36,8 @@ ${NEOGO} wallet nep17 transfer \
--from ${CONSADDR} --force \
--to ${SNADDR} \
--token GAS \
--amount 100

sleep 2 # https://github.com/nspcc-dev/neo-go/issues/3244
--amount 100 \
--await

set -m
/usr/bin/neofs-node --config /config/config-sn.yaml &
Expand All @@ -60,16 +54,6 @@ set -a

./bin/config.sh ContainerFee 0 && ./bin/config.sh ContainerAliasFee 0

if [ $IS_START_HTTP = "true" ]; then
. /config/http.env
/usr/bin/neofs-http-gw &

while [[ "$(curl -s -o /dev/null -w %{http_code} $HTTP_GW_SERVER_0_ADDRESS)" != "404" ]];
do
sleep 1;
done
fi

if [ $IS_START_REST = "true" ]; then
. /config/rest.env
/usr/bin/neofs-rest-gw &
Expand Down
5 changes: 1 addition & 4 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,11 @@ services:
restart: always
stop_signal: SIGKILL
env_file:
- "./http/http.env"
- "./rest-gw/rest.env"
volumes:
- ./sn/cli-cfg.yaml:/config/cli-cfg-sn.yaml
- ./sn/wallet.json:/config/wallet-sn.json
- ./sn/config.yaml:/config/config-sn.yaml
- ./http/wallet.json:/config/wallet-http.json
- ./rest-gw/wallet.json:/config/wallet-rest.json
- ./ir/cli-cfg.yaml:/config/cli-cfg-ir.yaml
- ./ir/config.yaml:/config/config-ir.yaml
Expand All @@ -25,7 +23,6 @@ services:
- "30333:30333" # RPC
- "8080:8080" # NeoFS API RPC
- "16513:16513" # Control service
- "8081:8081" # HTTP Gateway endpoint
- "8090:8090" # REST Gateway endpoint

nginx_gw:
Expand All @@ -36,7 +33,7 @@ services:
restart: on-failure
stop_signal: SIGKILL
volumes:
- ./http/nginx.conf:/etc/nginx/nginx.conf
- ./rest-gw/nginx.conf:/etc/nginx/nginx.conf
- cache:/cache
ports:
- "8082:8082" # nginx gateway endpoint
Expand Down
23 changes: 0 additions & 23 deletions http/http.env

This file was deleted.

30 changes: 0 additions & 30 deletions http/wallet.json

This file was deleted.

10 changes: 5 additions & 5 deletions http/nginx.conf → rest-gw/nginx.conf
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ http {
return 204;
}

rewrite '^(/[0-9a-zA-Z]{43,44}/[0-9a-zA-Z\-]{43,44})$' /get/$1 break;
rewrite '^/([0-9a-zA-Z]{43,44})/$' /get_by_attribute/$1/FileName/index.html break;
rewrite '^/([0-9a-zA-Z]{43,44})/([^/]*)$' /get_by_attribute/$1/FileName/$2 break;
rewrite '^/([0-9a-zA-Z]{43,44})/(.*)$' /get_by_attribute/$1/FilePath/$2 break;
rewrite '^(/[0-9a-zA-Z]{43,44}/[0-9a-zA-Z\-]{43,44})$' /v1/get/$1 break;
rewrite '^/([0-9a-zA-Z]{43,44})/$' /v1/get_by_attribute/$1/FileName/index.html break;
rewrite '^/([0-9a-zA-Z]{43,44})/([^/]*)$' /v1/get_by_attribute/$1/FileName/$2 break;
rewrite '^/([0-9a-zA-Z]{43,44})/(.*)$' /v1/get_by_attribute/$1/FilePath/$2 break;

proxy_pass http://aio:8081;
proxy_pass http://aio:8090;
proxy_http_version 1.1;
proxy_intercept_errors on;
proxy_buffering on;
Expand Down

0 comments on commit bb0dacd

Please sign in to comment.