diff --git a/docs/docs/guides/1_quickstart.md b/docs/docs/guides/1_quickstart.md index 306deb1b..90a1f517 100644 --- a/docs/docs/guides/1_quickstart.md +++ b/docs/docs/guides/1_quickstart.md @@ -10,6 +10,7 @@ This quick start will explore how to use Stencil command line interface and clie - [Docker](../installation#using-docker-image) or a [local installation](../installation#binary-cross-platform) of the Stencil binary. - A development environment applicable to one of the languages in this quick start (currently Go, Java, and JavaScript). - Postgres database and [protoc](https://github.com/protocolbuffers/protobuf#protocol-compiler-installation) if your schema format is protobuf. +- Kafka ## Step 1: Start server @@ -43,7 +44,7 @@ Note: Below command assumes `stencil_dev` db present in your postgres instance. $ docker run -e PORT=8000 -e DB_CONNECTIONSTRING=postgres://postgres@host.docker.internal:5432/stencil_dev?sslmode=disable -p 8000:8000 gotocompany/stencil server migrate # Stencil server at port 8000 -$ docker run -e PORT=8000 -e DB_CONNECTIONSTRING=postgres://postgres@host.docker.internal:5432/stencil_dev?sslmode=disable -p 8000:8000 gotocompany/stencil server start +$ docker run -e PORT=8000 -e KAFKAPRODUCER_BOORSTRAP_SERVER=localhost:9092 SCHEMACHANGE_ENABLE=true SCHEMACHANGE_KAFKATOPIC=schema_change DB_CONNECTIONSTRING=postgres://postgres@host.docker.internal:5432/stencil_dev?sslmode=disable -p 8000:8000 gotocompany/stencil server start # Check if server running $ curl -X GET http://localhost:8000/ping diff --git a/docs/docs/guides/3_manage_schemas.md b/docs/docs/guides/3_manage_schemas.md index cff64610..2babb81c 100644 --- a/docs/docs/guides/3_manage_schemas.md +++ b/docs/docs/guides/3_manage_schemas.md @@ -43,7 +43,7 @@ echo "syntax=\"proto3\";\npackage stencil;\nmessage One {\n int32 field_one = 2 protoc --descriptor_set_out=./file.desc --include_imports ./**/*.proto; # now try to upload this descriptor file with same name as before. This call should fail, giving you reason it has failed. -curl -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc"; +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc"; # now let's try fixing our proto add a new field without having any breaking changes. echo "syntax=\"proto3\";\npackage stencil;\nmessage One {\n int32 field_one = 1;\nint32 field_two = 2;\n}" > one.proto; @@ -52,7 +52,7 @@ echo "syntax=\"proto3\";\npackage stencil;\nmessage One {\n int32 field_one = 1 protoc --descriptor_set_out=./file.desc --include_imports ./**/*.proto # now try to upload this descriptor file with same name as before. This call should succeed -curl -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" ``` ## List versions @@ -63,5 +63,5 @@ curl -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binar curl -X GET http://localhost:8000/v1beta1/namespaces/quickstart/schemas/example/versions # upload schema can be called multiple times. Stencil server will retain old version if it's already uploaded. This call won't create new version again. You can verify by using versions API again. -curl -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" ``` diff --git a/docs/docs/guides/5_schema_change.md b/docs/docs/guides/5_schema_change.md new file mode 100644 index 00000000..45bf980d --- /dev/null +++ b/docs/docs/guides/5_schema_change.md @@ -0,0 +1,103 @@ +# Detect Schema Change + +## Enabling Schema Change + +To enable schema change toggle on the flag `SCHEMACHANGE_ENABLE` and provide `KAFKAPRODUCER_BOOTSTRAPSERVER` +and `SCHEMACHANGE_KAFKATOPIC`. + +## Create a schema + +```bash +# create namespace named "quickstart" +curl -X POST http://localhost:8000/v1beta1/namespaces -H 'Content-Type: application/json' -d '{"id": "quickstart", "format": "FORMAT_PROTOBUF", "compatibility": "COMPATIBILITY_BACKWARD", "description": "This field can be used to store namespace description"}' + +# create descriptor file v1 +protoc --descriptor_set_out=./protos/1.desc --include_imports ./guide/protos/1.proto + +It will be version 1 +# upload generated proto descriptor file to server with schema name as `example` under `quickstart` namespace. +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1beta1/namespaces/quickstart/schemas/example --data-binary "@1.desc" + +# create descriptor file v2 +protoc --descriptor_set_out=./protos/1.desc --include_imports ./guide/protos/2.proto + +It will be version 2 + +# upload generated proto descriptor file to server with schema name as `example` under `quickstart` namespace. +curl -X POST http://localhost:8000/v1beta1/namespaces/quickstart/schemas/example --data-binary "@2.desc" + +# Detect Schema Change +Now, when the schema is uploaded in DB. Below things happened parallely in separate go routine +1. It will get the previous schema data corresponding to that namespace and schema name. +2. It will check all the messages in that schema whether they are impacted(directly/indireclty) or not. +3. For change, it will check if any fileds got added/deleted. In this case only `address` field is added in `friend` message. So, `friend` proto is only impacted. +4. In this way it gets all impacted messages. And do below steps for all messages. +5. Then it will get the all indirectly impacted messages due to the message `friend`. +6. So, it will get `User` as it is using `Friend` message as compostion. +7. It will do the same thing for `User` also until all impacted messages got proccessed. +8. Then it will make `SchemChangedEvent` object and push it to a kafka topic. +9. After that it will update the `notification_events` database table with `success=true` after pushing the object to kafka. +``` + +## Handling Failures + +In case of any failure while publishing to Kafka or other issues, the `notification_events` table can be utilized to +manage these failures. If the `success` field is `false` for a specific `namespace_id`, `schema_id`, and `version_id`, +it indicates that the notification was not sent. In such scenarios, the `reconciliation API` can be used to resend the +notifications. + +## Depth + +In schema changes, we use the `depth` parameter to determine the level of impacted schemas to retrieve. If `depth` is +set to an empty string `""` or `-1`, it will fetch all indirectly impacted schemas. However, if a specific depth is +provided, it will only retrieve schemas up to that specified level. For example, if `depth` is set to `3`, it will fetch +up to three levels of impacted schemas, even if there are more levels of actual impacted schemas. +E.g. If depth is `2` then the `impacted_schemas` for `Address` in below sample would contain only `Address` and `Friend` message. + +## Sample SchemaChangedEvent Object + +``` +{ + "namespace_name": "quickstart", + "schema_name": "example", + "updated_schemas": [ + "test.stencil.main.Friend", + "test.stencil.main.Address", + ], + "updated_fields": { + "test.stencil.main.Friend": { + "field_names": [ + "address" + ] + }, + "test.stencil.main.Address": { + "field_names": [ + "city" + ] + } + }, + "impacted_schemas": { + "test.stencil.main.Address": { + "schema_names": [ + "test.stencil.main.Address", + "test.stencil.main.Friend", + "test.stencil.main.User", + ] + }, + "test.stencil.main.Friend": { + "schema_names": [ + "test.stencil.main.Friend", + "test.stencil.main.User", + ] + } + }, + "version": 1, + "metadata": { + "source_url": "https://github.com/some-repo", + "commit_sha": "some-commit-sha" + } +} +``` + +## Showing MR information +While calculating the impacted schemas, the `SchemaChangedEvent` will also include information about the source URL and commit SHA. The `source_url` represents the repository URL, and the `commit_sha` corresponds to the commit SHA associated with that version. diff --git a/docs/docs/guides/protos/1.proto b/docs/docs/guides/protos/1.proto new file mode 100644 index 00000000..d9b2a2fb --- /dev/null +++ b/docs/docs/guides/protos/1.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package gotocompany.test.stencil.main; + +option java_multiple_files = true; +option java_package = "com.test.stencil.main"; +option java_outer_classname = "FriendsProto"; + +message User { + string name = 1; + repeated Friend friends = 2; +} + +message Friend { + string name = 1; +} + diff --git a/docs/docs/guides/protos/2.proto b/docs/docs/guides/protos/2.proto new file mode 100644 index 00000000..7bcc9659 --- /dev/null +++ b/docs/docs/guides/protos/2.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package gotocompany.test.stencil.main; + +option java_multiple_files = true; +option java_package = "com.test.stencil.main"; +option java_outer_classname = "FriendsProto"; + +message User{ + string name = 1; + repeated Friend friends = 2; +} + +message Friend { + string name = 1; + Address address = 2; +} + +message Address { + string city = 1; +} + diff --git a/docs/docs/reference/api.md b/docs/docs/reference/api.md index 2cc151be..24048ea9 100644 --- a/docs/docs/reference/api.md +++ b/docs/docs/reference/api.md @@ -13,7 +13,7 @@ List names of namespaces ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | --------------------------------------------------------------- | +|---------|-------------------------------|-----------------------------------------------------------------| | 200 | A successful response. | [v1beta1ListNamespacesResponse](#v1beta1listnamespacesresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -26,13 +26,13 @@ Create namespace entry ##### Parameters | Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | --------------------------------------------------------------- | +|------|------------|-------------|----------|-----------------------------------------------------------------| | body | body | | Yes | [v1beta1CreateNamespaceRequest](#v1beta1createnamespacerequest) | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------------| | 200 | A successful response. | [v1beta1CreateNamespaceResponse](#v1beta1createnamespaceresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -47,13 +47,13 @@ Get namespace by id ##### Parameters | Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | +|------|------------|-------------|----------|--------| | id | path | | Yes | string | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------| | 200 | A successful response. | [v1beta1GetNamespaceResponse](#v1beta1getnamespaceresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -70,13 +70,13 @@ Ensure all schemas under this namespace is deleted, otherwise it will throw erro ##### Parameters | Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | +|------|------------|-------------|----------|--------| | id | path | | Yes | string | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------------| | 200 | A successful response. | [v1beta1DeleteNamespaceResponse](#v1beta1deletenamespaceresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -89,14 +89,14 @@ Update namespace entity by id ##### Parameters | Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | +|------|------------|-------------|----------|--------| | id | path | | Yes | string | | body | body | | Yes | object | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------------| | 200 | A successful response. | [v1beta1UpdateNamespaceResponse](#v1beta1updatenamespaceresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -111,13 +111,13 @@ List schemas under the namespace ##### Parameters | Name | Located in | Description | Required | Schema | -| ---- | ---------- | ----------- | -------- | ------ | +|------|------------|-------------|----------|--------| | id | path | | Yes | string | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | --------------------------------------------------------- | +|---------|-------------------------------|-----------------------------------------------------------| | 200 | A successful response. | [v1beta1ListSchemasResponse](#v1beta1listschemasresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -131,19 +131,20 @@ Get latest schema ##### Description -Returns latest schema in it's own data type. For protobuf response type would be 'application/octet-stream'. Avro, json schema response type would be 'application/json' +Returns latest schema in it's own data type. For protobuf response type would be 'application/octet-stream'. Avro, json +schema response type would be 'application/json' ##### Parameters | Name | Located in | Description | Required | Schema | -| ----------- | ---------- | ----------- | -------- | ------ | +|-------------|------------|-------------|----------|--------| | namespaceId | path | | Yes | string | | schemaId | path | | Yes | string | ##### Responses | Code | Description | Schema | -| ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | +|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------| | 200 | A successful schema response. Based on schema format, response will return different content types. For avro and json schemas response type is `application/json`. For protobuf response type is `application/octet-stream`. | | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -155,16 +156,20 @@ Create schema under the namespace ##### Parameters -| Name | Located in | Description | Required | Schema | -| ----------- | ---------- | --------------------------------------------------------------------- | -------- | ------ | -| namespaceId | path | | Yes | string | -| schemaId | path | | Yes | string | -| body | body | Request payload should be equivalent to `curl` `--data-binary` option | Yes | binary | +| Name | Located in | Description | Required | Schema | +|-----------------|------------|-----------------------------------------------------------------------|----------|--------| +| namespaceId | path | | Yes | string | +| schemaId | path | | Yes | string | +| body | body | Request payload should be equivalent to `curl` `--data-binary` option | Yes | binary | +| X-Format | header | | No | string | +| X-Compatibility | header | | No | string | +| X-SourceURL | header | source URL of the project | No | string | +| X-CommitSHA | header | commit SHA of corresponding to th change | No | string | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------| | 200 | A successful response. | [v1beta1CreateSchemaResponse](#v1beta1createschemaresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -177,14 +182,14 @@ Delete specified schema ##### Parameters | Name | Located in | Description | Required | Schema | -| ----------- | ---------- | ----------- | -------- | ------ | +|-------------|------------|-------------|----------|--------| | namespaceId | path | | Yes | string | | schemaId | path | | Yes | string | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------| | 200 | A successful response. | [v1beta1DeleteSchemaResponse](#v1beta1deleteschemaresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -197,7 +202,7 @@ Update only schema metadata ##### Parameters | Name | Located in | Description | Required | Schema | -| ----------- | ---------- | ----------- | -------- | ------ | +|-------------|------------|-------------|----------|--------| | namespaceId | path | | Yes | string | | schemaId | path | | Yes | string | | body | body | | Yes | object | @@ -205,7 +210,7 @@ Update only schema metadata ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | --------------------------------------------------------------------------- | +|---------|-------------------------------|-----------------------------------------------------------------------------| | 200 | A successful response. | [v1beta1UpdateSchemaMetadataResponse](#v1beta1updateschemametadataresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -224,7 +229,7 @@ Checks comptibility with existing latest schema ##### Parameters | Name | Located in | Description | Required | Schema | -| --------------- | ---------- | ----------- | -------- | ------ | +|-----------------|------------|-------------|----------|--------| | namespaceId | path | | Yes | string | | schemaId | path | | Yes | string | | body | body | | Yes | binary | @@ -233,7 +238,7 @@ Checks comptibility with existing latest schema ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------| | 200 | A successful response. | [v1beta1CreateSchemaResponse](#v1beta1createschemaresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -248,14 +253,14 @@ Create schema under the namespace. Returns version number, unique ID and locatio ##### Parameters | Name | Located in | Description | Required | Schema | -| ----------- | ---------- | ----------- | -------- | ------ | +|-------------|------------|-------------|----------|--------| | namespaceId | path | | Yes | string | | schemaId | path | | Yes | string | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | --------------------------------------------------------------------- | +|---------|-------------------------------|-----------------------------------------------------------------------| | 200 | A successful response. | [v1beta1GetSchemaMetadataResponse](#v1beta1getschemametadataresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -270,14 +275,14 @@ List all version numbers for schema ##### Parameters | Name | Located in | Description | Required | Schema | -| ----------- | ---------- | ----------- | -------- | ------ | +|-------------|------------|-------------|----------|--------| | namespaceId | path | | Yes | string | | schemaId | path | | Yes | string | ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------------------| | 200 | A successful response. | [v1beta1ListVersionsResponse](#v1beta1listversionsresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -292,7 +297,7 @@ Delete specified version of the schema ##### Parameters | Name | Located in | Description | Required | Schema | -| ----------- | ---------- | ----------- | -------- | ------- | +|-------------|------------|-------------|----------|---------| | namespaceId | path | | Yes | string | | schemaId | path | | Yes | string | | versionId | path | | Yes | integer | @@ -300,7 +305,7 @@ Delete specified version of the schema ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ------------------------------------------------------------- | +|---------|-------------------------------|---------------------------------------------------------------| | 200 | A successful response. | [v1beta1DeleteVersionResponse](#v1beta1deleteversionresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -315,7 +320,7 @@ Global Search API ##### Parameters | Name | Located in | Description | Required | Schema | -| ----------- | ---------- | ----------- | -------- | ------- | +|-------------|------------|-------------|----------|---------| | namespaceId | query | | No | string | | schemaId | query | | No | string | | query | query | | Yes | string | @@ -325,7 +330,7 @@ Global Search API ##### Responses | Code | Description | Schema | -| ------- | ----------------------------- | ----------------------------------------------- | +|---------|-------------------------------|-------------------------------------------------| | 200 | A successful response. | [v1beta1SearchResponse](#v1beta1searchresponse) | | default | An unexpected error response. | [rpcStatus](#rpcstatus) | @@ -334,26 +339,26 @@ Global Search API #### SchemaCompatibility | Name | Type | Description | Required | -| ------------------- | ------ | ----------- | -------- | +|---------------------|--------|-------------|----------| | SchemaCompatibility | string | | | #### SchemaFormat | Name | Type | Description | Required | -| ------------ | ------ | ----------- | -------- | +|--------------|--------|-------------|----------| | SchemaFormat | string | | | #### protobufAny | Name | Type | Description | Required | -| ------- | ------ | ----------- | -------- | +|---------|--------|-------------|----------| | typeUrl | string | | No | | value | byte | | No | #### rpcStatus | Name | Type | Description | Required | -| ------- | ------------------------------- | ----------- | -------- | +|---------|---------------------------------|-------------|----------| | code | integer | | No | | message | string | | No | | details | [ [protobufAny](#protobufany) ] | | No | @@ -361,13 +366,13 @@ Global Search API #### v1beta1CheckCompatibilityResponse | Name | Type | Description | Required | -| --------------------------------- | ------ | ----------- | -------- | +|-----------------------------------|--------|-------------|----------| | v1beta1CheckCompatibilityResponse | object | | | #### v1beta1CreateNamespaceRequest | Name | Type | Description | Required | -| ------------- | ------------------------------------------- | ----------- | -------- | +|---------------|---------------------------------------------|-------------|----------| | id | string | | Yes | | format | [SchemaFormat](#schemaformat) | | No | | compatibility | [SchemaCompatibility](#schemacompatibility) | | No | @@ -376,13 +381,13 @@ Global Search API #### v1beta1CreateNamespaceResponse | Name | Type | Description | Required | -| --------- | ------------------------------------- | ----------- | -------- | +|-----------|---------------------------------------|-------------|----------| | namespace | [v1beta1Namespace](#v1beta1namespace) | | No | #### v1beta1CreateSchemaResponse | Name | Type | Description | Required | -| -------- | ------- | ----------- | -------- | +|----------|---------|-------------|----------| | version | integer | | No | | id | string | | No | | location | string | | No | @@ -390,37 +395,37 @@ Global Search API #### v1beta1DeleteNamespaceResponse | Name | Type | Description | Required | -| ------- | ------ | ----------- | -------- | +|---------|--------|-------------|----------| | message | string | | No | #### v1beta1DeleteSchemaResponse | Name | Type | Description | Required | -| ------- | ------ | ----------- | -------- | +|---------|--------|-------------|----------| | message | string | | No | #### v1beta1DeleteVersionResponse | Name | Type | Description | Required | -| ------- | ------ | ----------- | -------- | +|---------|--------|-------------|----------| | message | string | | No | #### v1beta1GetLatestSchemaResponse | Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | +|------|------|-------------|----------| | data | byte | | No | #### v1beta1GetNamespaceResponse | Name | Type | Description | Required | -| --------- | ------------------------------------- | ----------- | -------- | +|-----------|---------------------------------------|-------------|----------| | namespace | [v1beta1Namespace](#v1beta1namespace) | | No | #### v1beta1GetSchemaMetadataResponse | Name | Type | Description | Required | -| ------------- | ------------------------------------------- | ----------- | -------- | +|---------------|---------------------------------------------|-------------|----------| | format | [SchemaFormat](#schemaformat) | | No | | compatibility | [SchemaCompatibility](#schemacompatibility) | | No | | authority | string | | No | @@ -428,31 +433,31 @@ Global Search API #### v1beta1GetSchemaResponse | Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | +|------|------|-------------|----------| | data | byte | | No | #### v1beta1ListNamespacesResponse | Name | Type | Description | Required | -| ---------- | ---------- | ----------- | -------- | +|------------|------------|-------------|----------| | namespaces | [ string ] | | No | #### v1beta1ListSchemasResponse | Name | Type | Description | Required | -| ------- | ---------- | ----------- | -------- | +|---------|------------|-------------|----------| | schemas | [ string ] | | No | #### v1beta1ListVersionsResponse | Name | Type | Description | Required | -| -------- | ----------- | ----------- | -------- | +|----------|-------------|-------------|----------| | versions | [ integer ] | | No | #### v1beta1Namespace | Name | Type | Description | Required | -| ------------- | ------------------------------------------- | ----------- | -------- | +|---------------|---------------------------------------------|-------------|----------| | id | string | | No | | format | [SchemaFormat](#schemaformat) | | No | | Compatibility | [SchemaCompatibility](#schemacompatibility) | | No | @@ -463,7 +468,7 @@ Global Search API #### v1beta1SearchHits | Name | Type | Description | Required | -| ----------- | ---------- | ----------- | -------- | +|-------------|------------|-------------|----------| | namespaceId | string | | No | | schemaId | string | | No | | versionId | integer | | No | @@ -474,26 +479,59 @@ Global Search API #### v1beta1SearchMeta | Name | Type | Description | Required | -| ----- | ---- | ----------- | -------- | +|-------|------|-------------|----------| | total | long | | No | #### v1beta1SearchResponse | Name | Type | Description | Required | -| ---- | ------------------------------------------- | ----------- | -------- | +|------|---------------------------------------------|-------------|----------| | hits | [ [v1beta1SearchHits](#v1beta1searchhits) ] | | No | | meta | [v1beta1SearchMeta](#v1beta1searchmeta) | | No | #### v1beta1UpdateNamespaceResponse | Name | Type | Description | Required | -| --------- | ------------------------------------- | ----------- | -------- | +|-----------|---------------------------------------|-------------|----------| | namespace | [v1beta1Namespace](#v1beta1namespace) | | No | #### v1beta1UpdateSchemaMetadataResponse | Name | Type | Description | Required | -| ------------- | ------------------------------------------- | ----------- | -------- | +|---------------|---------------------------------------------|-------------|----------| | format | [SchemaFormat](#schemaformat) | | No | | compatibility | [SchemaCompatibility](#schemacompatibility) | | No | | authority | string | | No | + +#### v1beta1SchemaChangedEvent + +| Name | Type | Description | Required | +|------------------|------------------------------------------------------------------------|----------------------------------------------------------------------|----------| +| event_id | string | event id of the message | No | +| event_timestamp | google.protobuf.Timestamp | timestamp for the message | No | +| namespace_name | string | name of namespace | No | +| schema_name | string | name of schema | No | +| updated_schemas | repeated string | directly updated schemas | No | +| updated_fields | [ImpactedFields](#v1beta1ImpactedFields)(map | impacted fields corresponding to schema | No | +| impacted_schemas | [ImpactedSchemas](#v1beta1ImpactedSchemas)map | indirectly impacted schema corresponding to directly impacted schema | No | +| version | int32 | version | No | +| metadata | [Metadata](v1beta1Metadata) | Metdata like commit_sha , source_url | No | + +#### v1beta1ImpactedFields + +| Name | Type | Description | Required | +|-------------|-----------------|------------------------------|----------| +| field_names | repeated string | List of impacted field names | No | + +#### v1beta1ImpactedSchemas + +| Name | Type | Description | Required | +|--------------|-----------------|-------------------------------|----------| +| schema_names | repeated string | List of impacted schema names | No | + +#### v1beta1Metadata + +| Name | Type | Description | Required | +|------------|--------|---------------------------|----------| +| source_url | string | source URL of the project | No | +| commit_sha | string | commit SHA of the version | No | \ No newline at end of file diff --git a/docs/docs/server/overview.md b/docs/docs/server/overview.md index 8d16484c..483822fd 100644 --- a/docs/docs/server/overview.md +++ b/docs/docs/server/overview.md @@ -13,6 +13,7 @@ Stencil is dynamic protobuf schema registry. It provides REST interface for stor ## Requirements - postgres 13 +- kafka 3.7.0 ## Installation @@ -42,14 +43,21 @@ $ ./stencil server start -c config.yaml You can also specfify stencil server configurations through following environment variables. Note: ENV vars takes more precendence over config file. -| ENV | Description | -| :-------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PORT` | port number default to `8080` | -| `TIMEOUT` | graceful time to wait before shutting down the server. Takes `time.Duration` format. Eg: `30s` or `20m` | -| `DB_CONNECTIONSTRING` | postgres db connection [url](https://www.postgresql.org/docs/11/libpq-connect.html#LIBPQ-CONNSTRING). Eg: `postgres://postgres@localhost:5432/db_name` | -| `NEWRELIC_ENABLED` | boolean to enable newrelic | -| `NEWRELIC_APPNAME` | appname | -| `NEWRELIC_LICENSE` | License key for newrelic | +| ENV | Description | +|:--------------------------------| :----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PORT` | port number default to `8080` | +| `TIMEOUT` | graceful time to wait before shutting down the server. Takes `time.Duration` format. Eg: `30s` or `20m` | +| `DB_CONNECTIONSTRING` | postgres db connection [url](https://www.postgresql.org/docs/11/libpq-connect.html#LIBPQ-CONNSTRING). Eg: `postgres://postgres@localhost:5432/db_name` | +| `NEWRELIC_ENABLED` | boolean to enable newrelic | +| `NEWRELIC_APPNAME` | appname | +| `NEWRELIC_LICENSE` | License key for newrelic +| `KAFKAPRODUCER_BOOTSTRAPSERVER` | bootstrap server for kafka +| `KAFKAPRODUCER_RETRIES` | Number of retries for producer if publish to kafka topic fails +| `KAFKAPRODUCER_TIMEOUT` | Timeout for write operation performed by the Writer. +| `SCHEMACHANGE_ENABLE` | flag to enable change detector +| `SCHEMACHANGE_KAFKATOPIC` | name of kafka topic to which messages related to SchemaChangeEvent will be published +| `SCHEMACHANGE_DEPTH` | Depth to which indirectly impacted schemas are processed. If empty or -1, it will process all indirectly impacted schemas. + ## Reference @@ -76,7 +84,7 @@ curl -X POST http://localhost:8000/v1beta1/namespaces -H 'Content-Type: applicat curl http://localhost:8000/v1beta1/namespaces # upload generated proto descriptor file to server with schema name as `example` under `quickstart` namespace. -curl -X POST http://localhost:8000/v1beta1/namespaces/quickstart/schemas/example --data-binary "@file.desc" +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1beta1/namespaces/quickstart/schemas/example --data-binary "@file.desc" # get list of schemas available in a namespace curl -X GET http://localhost:8000/v1beta1/namespaces/quickstart/schemas @@ -97,7 +105,7 @@ echo "syntax=\"proto3\";\npackage stencil;\nmessage One {\n int32 field_one = 2 protoc --descriptor_set_out=./file.desc --include_imports ./**/*.proto; # now try to upload this descriptor file with same name as before. This call should fail, giving you reason it has failed. -curl -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc"; +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc"; # now let's try fixing our proto add a new field without having any breaking changes. echo "syntax=\"proto3\";\npackage stencil;\nmessage One {\n int32 field_one = 1;\nint32 field_two = 2;\n}" > one.proto; @@ -106,11 +114,11 @@ echo "syntax=\"proto3\";\npackage stencil;\nmessage One {\n int32 field_one = 1 protoc --descriptor_set_out=./file.desc --include_imports ./**/*.proto # now try to upload this descriptor file with same name as before. This call should succeed -curl -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" # now try versions api. It should have 2 versions now. curl -X GET http://localhost:8000/v1beta1/namespaces/quickstart/schemas/example/versions # upload schema can be called multiple times. Stencil server will retain old version if it's already uploaded. This call won't create new version again. You can verify by using versions API again. -curl -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" +curl -H "X-SourceURL:www.github.com/some-repo" -H "X-CommitSHA:some-commit-sha" -X POST http://localhost:8000/v1/namespaces/quickstart/schemas --data-binary "@file.desc" ```