Merge pull request #746 from wireapp/release/2019-05-02

Release/2019 05 02

CHANGELOG.md

@@ -1,3 +1,33 @@
+# 2019-04-09 #746
+
+## Documentation changes
+
+* Improved Cassandra documentation in `docs/README.md`
+* Improved documentation on SCIM storage in `docs/README.md`
+* Improved documentation on SCIM Tokens in `docs/reference/provisioning/scim-token.md`
+
+## Bug fixes
+
+* Sanitize metric names to be valid prometheus names in metrics-core
+* Add missing a `.git` suffix on gitlab dependencies in stack.yaml
+* Time bounds checks now allow 60s of tolerance; this is helpful in cases of drifting clocks (#730)
+
+## Features
+
+* Services now provide Prometheus metrics on `/i/metrics`
+* Garbage Collection and memory statistics are available alongside other prometheus metrics
+
+## Internal Changes
+
+* Alpine Builder is no longer built with `--profile`
+* SCIM users now have an additional wire-specific schema attached.
+
+## Changes (potentially) requiring action
+* `/i/monitoring` is *DEPRECATED*. Please use prometheus metrics provided by `/i/metrics` instead.
+* On password reset the new password must be different than the old one
+* Stern is now available as a new tool for performing adminstrative tasks via API (#720)
+* SCIM handler errors are now reported according to SCIM error schema (#575)
+
 # 2019-04-09 #710
 
 ## API changes

Makefile

@@ -8,35 +8,41 @@ default: fast
 init:
  mkdir -p dist
 
+# Build all Haskell services and executables, run unit tests
 .PHONY: install
 install: init
  stack install --pedantic --test --bench --no-run-benchmarks --local-bin-path=dist
 
+# Build all Haskell services and executables with -O0, run unit tests
 .PHONY: fast
 fast: init
  stack install --pedantic --test --bench --no-run-benchmarks --local-bin-path=dist --fast $(WIRE_STACK_OPTIONS)
 
-.PHONY: clean
-clean:
- stack clean
- $(MAKE) -C services/nginz clean
- -rm -rf dist
- -rm -f .metadata
-
+# Build everything (Haskell services and nginz)
 .PHONY: services
 services: init install
  $(MAKE) -C services/nginz
 
+# Build haddocks
 .PHONY: haddock
 haddock:
  WIRE_STACK_OPTIONS="--haddock --haddock-internal" make fast
 
+# Build haddocks only for wire-server
 .PHONY: haddock-shallow
 haddock-shallow:
  WIRE_STACK_OPTIONS="--haddock --haddock-internal --no-haddock-deps" make fast
 
+# Clean
+.PHONY: clean
+clean:
+ stack clean
+ $(MAKE) -C services/nginz clean
+ -rm -rf dist
+ -rm -f .metadata
+
 #################################
-## integration tests
+## running integration tests
 
 # Build services with --fast and run tests
 .PHONY: integration

README.md

@@ -54,6 +54,7 @@ This repository contains the following source code:
  - **makedeb**: Create Debian packages
  - **bonanza**: Transform and forward log data
  - **db/**: Migration tools (e.g. when new tables are added)
+ - **stern/**: Backoffice tool (basic [Swagger](https://swagger.io/) based interface)
 
 - **libs**: Shared libraries
 

build/alpine/Dockerfile.builder

@@ -8,12 +8,8 @@ WORKDIR /
 # Download stack indices and compile/cache dependencies to speed up subsequent
 # container creation.
 #
-# We also build profiling versions of all libraries. Due to a bug in Stack,
-# they have to be built in a separate directory. See this issue:
-# https://github.com/commercialhaskell/stack/issues/4032
-#
-# Finally, we build docs for haskell-src-exts without hyperlinking enabled
-# to avoid a Haddock segfault. See https://github.com/haskell/haddock/issues/928
+# We build docs for haskell-src-exts without hyperlinking enabled to avoid
+# a Haddock segfault. See https://github.com/haskell/haddock/issues/928
 #
 # Note: git, ncurses, sed are added here for historical reasons; since
 # roughly 2019-03-28, they are included in prebuilder as well.
@@ -23,9 +19,9 @@ RUN apk add --no-cache git ncurses sed && \
  cd /wire-server && \
  stack update && \
  echo "allow-different-user: true" >> /root/.stack/config.yaml && \
- stack build --haddock --dependencies-only --profile haskell-src-exts && \
- stack build --haddock --no-haddock-hyperlink-source --profile haskell-src-exts && \
- stack build --pedantic --haddock --test --no-run-tests --bench --no-run-benchmarks --dependencies-only --profile && \
+ stack build --haddock --dependencies-only haskell-src-exts && \
+ stack build --haddock --no-haddock-hyperlink-source haskell-src-exts && \
+ stack build --pedantic --haddock --test --no-run-tests --bench --no-run-benchmarks --dependencies-only && \
  cd / && \
  # we run the build only to cache the built source in /root/.stack, we can remove the source code itself
  rm -rf /wire-server

deploy/services-demo/conf/spar.demo-docker.yaml

@@ -31,7 +31,7 @@ cassandra:
 maxttlAuthreq: 28800 # 8h
 maxttlAuthresp: 28800 # 8h
 
-maxScimTokens: 16
+maxScimTokens: 16 # Token limit {#RefScimToken}
 richInfoLimit: 5000 # should be in sync with Brig
 
 logNetStrings: False # log using netstrings encoding (see http://cr.yp.to/proto/netstrings.txt)

deploy/services-demo/conf/spar.demo.yaml

@@ -31,7 +31,7 @@ cassandra:
 maxttlAuthreq: 28800 # 8h
 maxttlAuthresp: 28800 # 8h
 
-maxScimTokens: 16
+maxScimTokens: 16 # Token limit {#RefScimToken}
 richInfoLimit: 5000 # should be in sync with Brig
 
 logNetStrings: False # log using netstrings encoding (see http://cr.yp.to/proto/netstrings.txt)

deploy/services-demo/create_test_user.sh

@@ -3,14 +3,14 @@
 set -e
 
 #
-# This bash script can be used to create an active user by using an internal 
+# This bash script can be used to create an active user by using an internal
 # brig endpoint. Note that this is not exposed over nginz and can only be used
 # if you have direct access to brig
 #
 
 # Usage:
-# --csv Output users in CSV format
-# --count=INT Generate several users (by default it's just one)
+# --csv Output users in CSV format
+# --count=INT Generate several users (by default it's just one)
 
 CSV=false
 COUNT=1
@@ -47,7 +47,7 @@ do
  -H'Content-type: application/json' \
  -d'{"email":"'$EMAIL'","password":"'$PASSWORD'","name":"demo"}')
 
- UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*"id":"\([0-9a-z-]\+\)".*/\1/')
+ UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
 
  if [ "$CSV" == "false" ]
  then echo -e "Succesfully created a user with email: "$EMAIL" and password: "$PASSWORD

docs/README.md

@@ -1,26 +1,42 @@
-(incomplete)
-
 # Reference documentation
 
 What you need to know as a user of the Wire backend: concepts, features, and API. We strive to keep these up to date.
 
 ## Users
 
-We support the following functionality related to users and user profiles:
+User lifecycle:
+
+* [User registration](reference/user/registration.md) `{#RefRegistration}`
+* [User activation](reference/user/activation.md) `{#RefActivation}`
+
+User profiles and metadata:
 
 * [Connections between users](reference/user/connection.md) `{#RefConnection}`
 * [Rich info](reference/user/rich-info.md) `{#RefRichInfo}`
-* TODO
 
-## Provisioning
+TODO.
+
+## Teams
+
+TODO.
+
+## Messaging
+
+TODO.
+
+## Single sign-on
+
+TODO.
+
+## SCIM provisioning
 
 We have support for provisioning users via SCIM ([RFC 7664][], [RFC 7643][]). It's in the beta stage.
 
 [RFC 7664]: https://tools.ietf.org/html/rfc7664
 [RFC 7643]: https://tools.ietf.org/html/rfc7643
 
 * [Using the SCIM API with curl](reference/provisioning/scim-via-curl.md) `{#RefScimViaCurl}`
-* TODO
+* [Authentication via SCIM tokens](reference/provisioning/scim-token.md) `{#RefScimToken}`
 
 # Developer documentation
 
@@ -30,4 +46,18 @@ If you're not a member of the Wire backend team, you might still find these docu
 
 * [Development setup](developer/dependencies.md) `{#DevDeps}`
 * [Editor setup](developer/editor-setup.md) `{#DevEditor}`
+* [Storing SCIM-related data](developer/scim/storage.md) `{#DevScimStorage}`
 * TODO
+
+## Cassandra
+
+We use [Cassandra](http://cassandra.apache.org/) as the primary data store. It is scalable, has very fast reads and writes, and is conceptually simple (or at least simpler than SQL databases).
+
+Some helpful links:
+
+* [Query syntax](https://docs.datastax.com/en/cql/3.3/cql/cql_reference/cqlReferenceTOC.html)
+
+* How deletes work in Cassandra:
+
+ - [Understanding Deletes](https://medium.com/@foundev/domain-modeling-around-deletes-1cc9b6da0d24)
+ - [Cassandra Compaction and Tombstone Behavior](http://engblog.polyvore.com/2015/03/cassandra-compaction-and-tombstone.html)

docs/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-slate

docs/developer/scim/storage.md

@@ -0,0 +1,56 @@
+# Storing SCIM-related data {#DevScimStorage}
+
+_Author: Artyom Kazak_
+
+---
+
+## Storing user data {#DevScimStorageUsers}
+
+SCIM user data is stored as JSON blobs in the `scim_user` table in Spar, one blob per SCIM-managed user. Those blobs conform to the SCIM standard and are returned by `GET /scim/v2/Users`.
+
+Note that when a user is created via SCIM, the received blob is not written verbatim to the database – it is first parsed by the [hscim](https://github.com/wireapp/hscim) library, and all unknown fields are removed.
+
+Sample blob:
+
+```json
+{
+ "schemas": [
+ "urn:ietf:params:scim:schemas:core:2.0:User",
+ "urn:wire:scim:schemas:profile:1.0"
+ ],
+ "id": "ef4bafda-5be8-46e3-bed2-5bcce55cff01",
+ "externalId": "[email protected]",
+ "userName": "lana_d",
+ "displayName": "Lana Donohue",
+ "urn:wire:scim:schemas:profile:1.0": {
+ "richInfo": {
+ "version": 0,
+ "fields": [
+ { "type": "Title", "value": "Chief Backup Officer" },
+ { "type": "Favorite quote", "value": "Monads are just giant burritos" }
+ ]
+ }
+ },
+ "meta": {
+ "resourceType": "User",
+ "location": "https://staging-nginz-https.zinfra.io/scim/v2/Users/ef4bafda-5be8-46e3-bed2-5bcce55cff01",
+ "created": "2019-04-21T04:15:12.535509602Z",
+ "lastModified": "2019-04-21T04:15:18.185055531Z",
+ "version": "W/\"e051bc17f7e07dec815f4b9314f76f88e2949a62b6aad8c816086cff85de4783\""
+ }
+}
+```
+
+### One-way sync from Spar to Brig {#DevScimOneWaySync}
+
+A user is considered SCIM-managed if they were provisioned with SCIM (when it's the case, `userManagedBy` will be set to `ManagedByScim`). Data about SCIM-managed users is stored both in Brig and Spar, and should always be in sync.
+
+Currently (2019-04-29) we only implement one-way sync – when a user is modified via SCIM, Spar takes care to update data in Brig. However, user data is _not_ updated on the Spar side when it is changed in Brig, and Brig does not yet prohibit changing user data via its API – it relies on clients to be well-behaved and respect `userManagedBy`.
+
+## Storing SCIM tokens {#DevScimStorageTokens}
+
+[SCIM tokens](../../reference/provisioning/scim-token.md) are stored in two tables in Spar:
+
+* `team_provisioning_by_token` for `token -> token info` lookups; used to perform authentication.
+
+* `team_provisioning_by_team` for `team -> [token info]` and `(team, token ID) -> token info` lookups; used to display tokens in team settings, and to decide which tokens should be deleted when the whole team is deleted.