Merge pull request #3505 from wireapp/release_2023-08-16_09_54

Release 2023-08-16 - (expected chart version 4.37.0)

CHANGELOG.md

@@ -1,3 +1,23 @@
+# [2023-08-16] (Chart Release 4.37.0)
+
+## API changes
+
+
+* Conversation creation endpoints can now return `unreachable_backends` error responses with status code 533 if any of the involved backends are unreachable. The conversation is not created in that case. (#3486)
+
+
+## Bug fixes and other updates
+
+
+* Make sure cassandra updates do not re-introduce removed content. (#3504)
+
+
+## Federation changes
+
+
+* Return `unreachable_backends` error when some backends of newly added users to a conversation are not reachable (#3496)
+
+
 # [2023-08-11] (Chart Release 4.36.0)
 
 ## Release notes

hack/bin/create-user

@@ -64,8 +64,12 @@ def add_team_member(baseurl, team, access_token, basic_auth, i=1):
 
  return member
 
-def create_user(baseurl, basic_auth, create_team, n_members):
- email = random_email()
+def create_user(baseurl, basic_auth, create_team, n_members, manual_email, has_inbucket):
+ if manual_email is None:
+ email = random_email()
+ else:
+ email = manual_email
+
  password = random_string(20)
 
  body = {
@@ -99,25 +103,37 @@ def create_user(baseurl, basic_auth, create_team, n_members):
  'team': team
  }
 
- r = requests.post(f'{baseurl}/login', json={'email': email, 'password': password})
- access_token = r.json()['access_token']
-
  result = {'admin': admin}
 
- if team is not None:
- members = []
- for i in range(n_members):
- member = add_team_member(baseurl, team, access_token, basic_auth, i)
- members.append(member)
- result['members'] = members
+ r = requests.get(f'{baseurl}/i/teams/{team}/features/sndFactorPasswordChallenge', headers=basicauth_headers)
+ d = r.json()
+ second_factor_enabled = d['status'] == 'enabled'
+ # FUTUREWORK: Create team members for 2fa backends. To login 1) send verification code 2) get verification code via internal api 3) use code when logging in as authentication code
+ if second_factor_enabled:
+ if manual_email is None and not has_inbucket:
+ fail("Backend has 2FA enabled. Yout must provide an existing email adress via the -m flag. Also no team members will be created by this script.")
+
+ else:
+ login_request = {'email': email, 'password': password}
+
+ r = requests.post(f'{baseurl}/login', json=login_request)
+
+ access_token = r.json()['access_token']
+
+ if team is not None and not second_factor_enabled:
+ members = []
+ for i in range(n_members):
+ member = add_team_member(baseurl, team, access_token, basic_auth, i)
+ members.append(member)
+ result['members'] = members
 
  return result
 
-def maybe_to_list(x):
- if x is not None:
-  return [x]
- else:
- return []
+def fail(msg):
+ sys.stderr.write(msg)
+ sys.stderr.write('\n')
+ sys.exit(1)
+
 
 def main():
  known_envs = {
@@ -172,6 +188,37 @@ def main():
  'baseurl': 'https://nginz-https.unicorns.dogfood.wire.link',
  'webapp': 'https://webapp.unicorns.dogfood.wire.link/'
  },
+ 'bund-next-column-offline-android': {
+ 'baseurl': 'https://nginz-https.bund-next-column-offline-android.wire.link',
+ 'webapp': 'https://webapp.bund-next-column-offline-android.wire.link/'
+ },
+ 'bund-next-column-offline-web': {
+ 'baseurl': 'https://nginz-https.bund-next-column-offline-web.wire.link',
+ 'webapp': 'https://webapp.bund-next-column-offline-web.wire.link/'
+ },
+ 'bund-next-column-offline-ios': {
+ 'baseurl': 'https://nginz-https.bund-next-column-offline-ios.wire.link',
+ 'webapp': 'https://webapp.bund-next-column-offline-ios.wire.link/'
+ },
+ 'bund-next-external': {
+ 'baseurl': 'https://nginz-https.bund-next-external.wire.link',
+ 'webapp': 'https://webapp.bund-next-external.wire.link/'
+ },
+ 'bund-next-column-1': {
+ 'baseurl': 'https://nginz-https.bund-next-column-1.wire.link',
+ 'webapp': 'https://webapp.bund-next-column-1.wire.link/',
+ 'inbucket': 'https://inbucket.bund-next-column-1.wire.link/'
+ },
+ 'bund-next-column-2': {
+ 'baseurl': 'https://nginz-https.bund-next-column-2.wire.link',
+ 'webapp': 'https://webapp.bund-next-column-2.wire.link/',
+ 'inbucket': 'https://inbucket.bund-next-column-2.wire.link/'
+ },
+ 'bund-next-column-3': {
+ 'baseurl': 'https://nginz-https.bund-next-column-3.wire.link',
+ 'webapp': 'https://webapp.bund-next-column-3.wire.link/',
+ 'inbucket': 'https://inbucket.bund-next-column-3.wire.link/'
+ }
  }
 
  parser = argparse.ArgumentParser(
@@ -180,35 +227,37 @@ def main():
  parser.add_argument('-e', '--env', default='choose_env', help=f'One of: {", ".join(known_envs.keys())}')
  parser.add_argument('-p', '--personal', action='store_true', help="Create a personal user, instead of a team admin.")
  parser.add_argument('-n', '--members', default='1', help="Number of members to add.")
+ parser.add_argument('-m', '--email', default='', help="Email of created user. If omitted a random non-existing @wire.com email will be used.")
  args = parser.parse_args()
 
  if args.env == 'choose_env':
- print(parser.format_help())
- sys.exit(1)
+ fail(parser.format_help())
 
  env = known_envs.get(args.env)
  if env is None:
- print(f'Unknown environment: {args.env}. If missing then add it to the script.')
- sys.exit(1)
+ fail(f'Unknown environment: {args.env}. If missing then add it to the script.')
 
  basic_auths_json = os.environ.get('CREATE_USER_BASICAUTH')
  if basic_auths_json is None:
- print(r'Please set CREATE_USER_BASICAUTH to a json object of form {"env_name": {"username": "xx", "password": "xx"}} containing the basicauth credentials for each environment.')
- sys.exit(1)
+ fail(r'Please set CREATE_USER_BASICAUTH to a json object of form {"env_name": {"username": "xx", "password": "xx"}} containing the basicauth credentials for each environment.')
 
  basic_auths = json.loads(basic_auths_json)
+ if args.env not in basic_auths:
+ fail(f'Environment "{args.env}" is missing in CREATE_USER_BASICAUTH.')
+
  b_user = basic_auths[args.env]['username']
  b_password = basic_auths[args.env]['password']
 
  basic_auth = base64.b64encode(f'{b_user}:{b_password}'.encode('utf8')).decode('utf8')
 
  n_members = int(args.members)
 
- result = create_user(env['baseurl'], basic_auth, not args.personal, n_members)
+ manual_email = args.email if len(args.email) > 0 else None
+
+ result = create_user(env['baseurl'], basic_auth, not args.personal, n_members, manual_email, 'inbucket' in env)
 
- links = maybe_to_list(env.get('webapp')) + maybe_to_list(env.get('teams'))
- if links:
- result['comment'] = f'These credentials can be used at: {", ".join(links)}'
+ result['env'] = env
+ result['basicauth'] = {'username': b_user, 'password': b_password, 'header': basic_auth}
 
  print(json.dumps(result, indent=4))
 

integration/test/Test/Conversation.hs

@@ -51,8 +51,8 @@ testDynamicBackendsNotFederating = do
  $ bindResponse
  (getFederationStatus uidA [domainB, domainC])
  $ \resp -> do
- resp.status `shouldMatchInt` 422
- resp.json %. "label" `shouldMatch` "federation-denied"
+ resp.status `shouldMatchInt` 533
+ resp.json %. "unreachable_backends" `shouldMatchSet` [domainB, domainC]
 
 testDynamicBackendsFullyConnectedWhenAllowDynamic :: HasCallStack => App ()
 testDynamicBackendsFullyConnectedWhenAllowDynamic = do
@@ -123,8 +123,8 @@ testFederationStatus = do
  bindResponse
  (getFederationStatus uid [invalidDomain])
  $ \resp -> do
- resp.status `shouldMatchInt` 422
- resp.json %. "label" `shouldMatch` "invalid-domain"
+ resp.status `shouldMatchInt` 533
+ resp.json %. "unreachable_backends" `shouldMatchSet` [invalidDomain]
 
  bindResponse
  (getFederationStatus uid [federatingRemoteDomain])
@@ -327,3 +327,64 @@ testAddMembersNonFullyConnectedProteus = do
  bindResponse (addMembers u1 cid members) $ \resp -> do
  resp.status `shouldMatchInt` 409
  resp.json %. "non_federating_backends" `shouldMatchSet` [domainB, domainC]
+
+testConvWithUnreachableRemoteUsers :: HasCallStack => App ()
+testConvWithUnreachableRemoteUsers = do
+ let overrides =
+ def {dbBrig = setField "optSettings.setFederationStrategy" "allowAll"}
+ <> fullSearchWithAll
+ ([alice, alex, bob, charlie, dylan], domains) <-
+ startDynamicBackends [overrides, overrides] $ \domains -> do
+ own <- make OwnDomain & asString
+ other <- make OtherDomain & asString
+ users <- createAndConnectUsers $ [own, own, other] <> domains
+ pure (users, domains)
+
+ let newConv = defProteus {qualifiedUsers = [alex, bob, charlie, dylan]}
+ bindResponse (postConversation alice newConv) $ \resp -> do
+ resp.status `shouldMatchInt` 533
+ resp.json %. "unreachable_backends" `shouldMatchSet` domains
+
+ convs <- getAllConvs alice >>= asList
+ regConvs <- filterM (\c -> (==) <$> (c %. "type" & asInt) <*> pure 0) convs
+ regConvs `shouldMatch` ([] :: [Value])
+
+testAddReachableWithUnreachableRemoteUsers :: HasCallStack => App ()
+testAddReachableWithUnreachableRemoteUsers = do
+ let overrides =
+ def {dbBrig = setField "optSettings.setFederationStrategy" "allowAll"}
+ <> fullSearchWithAll
+ ([alex, bob], conv) <-
+ startDynamicBackends [overrides, overrides] $ \domains -> do
+ own <- make OwnDomain & asString
+ other <- make OtherDomain & asString
+ [alice, alex, bob, charlie, dylan] <-
+ createAndConnectUsers $ [own, own, other] <> domains
+
+ let newConv = defProteus {qualifiedUsers = [alex, charlie, dylan]}
+ conv <- postConversation alice newConv >>= getJSON 201
+ pure ([alex, bob], conv)
+
+ bobId <- bob %. "qualified_id"
+ bindResponse (addMembers alex conv [bobId]) $ \resp -> do
+ resp.status `shouldMatchInt` 200
+
+testAddUnreachable :: HasCallStack => App ()
+testAddUnreachable = do
+ let overrides =
+ def {dbBrig = setField "optSettings.setFederationStrategy" "allowAll"}
+ <> fullSearchWithAll
+ ([alex, charlie], [charlieDomain, _dylanDomain], conv) <-
+ startDynamicBackends [overrides, overrides] $ \domains -> do
+ own <- make OwnDomain & asString
+ [alice, alex, charlie, dylan] <-
+ createAndConnectUsers $ [own, own] <> domains
+
+ let newConv = defProteus {qualifiedUsers = [alex, dylan]}
+ conv <- postConversation alice newConv >>= getJSON 201
+ pure ([alex, charlie], domains, conv)
+
+ charlieId <- charlie %. "qualified_id"
+ bindResponse (addMembers alex conv [charlieId]) $ \resp -> do
+ resp.status `shouldMatchInt` 533
+ resp.json %. "unreachable_backends" `shouldMatchSet` [charlieDomain]

integration/test/Testlib/App.hs

@@ -8,13 +8,14 @@ import Data.IORef
 import Data.Text qualified as T
 import Data.Yaml qualified as Yaml
 import GHC.Exception
+import GHC.Stack (HasCallStack)
 import System.FilePath
 import Testlib.Env
 import Testlib.JSON
 import Testlib.Types
 import Prelude
 
-failApp :: String -> App a
+failApp :: HasCallStack => String -> App a
 failApp msg = throw (AppFailure msg)
 
 getPrekey :: App Value

integration/test/Testlib/HTTP.hs

@@ -82,18 +82,18 @@ withResponse :: HasCallStack => Response -> (Response -> App a) -> App a
 withResponse r k = onFailureAddResponse r (k r)
 
 -- | Check response status code, then return body.
-getBody :: Int -> Response -> App ByteString
+getBody :: HasCallStack => Int -> Response -> App ByteString
 getBody status resp = withResponse resp $ \r -> do
  r.status `shouldMatch` status
  pure r.body
 
 -- | Check response status code, then return JSON body.
-getJSON :: Int -> Response -> App Aeson.Value
+getJSON :: HasCallStack => Int -> Response -> App Aeson.Value
 getJSON status resp = withResponse resp $ \r -> do
  r.status `shouldMatch` status
  r.json
 
-onFailureAddResponse :: Response -> App a -> App a
+onFailureAddResponse :: HasCallStack => Response -> App a -> App a
 onFailureAddResponse r m = App $ do
  e <- ask
  liftIO $ E.catch (runAppWithEnv e m) $ \(AssertionFailure stack _ msg) -> do

integration/test/Testlib/ModService.hs

@@ -83,8 +83,8 @@ copyDirectoryRecursively from to = do
 -- continuation, the main continuation is run in an environment that
 -- accumulates all the individual environment changes.
 traverseConcurrentlyCodensity ::
- (a -> Codensity App (Env -> Env)) ->
- ([a] -> Codensity App (Env -> Env))
+ (HasCallStack => a -> Codensity App (Env -> Env)) ->
+ (HasCallStack => [a] -> Codensity App (Env -> Env))
 traverseConcurrentlyCodensity f args = do
  -- Create variables for synchronisation of the various threads:
  -- * @result@ is used to store the environment change, or possibly an exception
@@ -138,15 +138,19 @@ traverseConcurrentlyCodensity f args = do
  liftIO $ traverse_ wait asyncs
  pure result
 
-startDynamicBackends :: [ServiceOverrides] -> ([String] -> App a) -> App a
-startDynamicBackends beOverrides = runCodensity $ do
- when (Prelude.length beOverrides > 3) $ lift $ failApp "Too many backends. Currently only 3 are supported."
- pool <- asks (.resourcePool)
- resources <- acquireResources (Prelude.length beOverrides) pool
- void $ traverseConcurrentlyCodensity (\(res, overrides) -> startDynamicBackend res mempty overrides) (zip resources beOverrides)
- pure $ map (.berDomain) resources
+startDynamicBackends :: HasCallStack => [ServiceOverrides] -> (HasCallStack => [String] -> App a) -> App a
+startDynamicBackends beOverrides k =
+ runCodensity
+ ( do
+ when (Prelude.length beOverrides > 3) $ lift $ failApp "Too many backends. Currently only 3 are supported."
+ pool <- asks (.resourcePool)
+ resources <- acquireResources (Prelude.length beOverrides) pool
+ void $ traverseConcurrentlyCodensity (\(res, overrides) -> startDynamicBackend res mempty overrides) (zip resources beOverrides)
+ pure $ map (.berDomain) resources
+ )
+ k
 
-startDynamicBackend :: BackendResource -> Map.Map Service Word16 -> ServiceOverrides -> Codensity App (Env -> Env)
+startDynamicBackend :: HasCallStack => BackendResource -> Map.Map Service Word16 -> ServiceOverrides -> Codensity App (Env -> Env)
 startDynamicBackend resource staticPorts beOverrides = do
  defDomain <- asks (.domain1)
  let services =

integration/test/Testlib/ResourcePool.hs

@@ -21,6 +21,7 @@ import Data.String
 import Data.Tuple
 import Data.Word
 import GHC.Generics
+import GHC.Stack (HasCallStack)
 import System.IO
 import Prelude
 
@@ -29,7 +30,7 @@ data ResourcePool a = ResourcePool
  resources :: IORef (Set.Set a)
  }
 
-acquireResources :: forall m a. (Ord a, MonadIO m, MonadMask m) => Int -> ResourcePool a -> Codensity m [a]
+acquireResources :: forall m a. (Ord a, MonadIO m, MonadMask m, HasCallStack) => Int -> ResourcePool a -> Codensity m [a]
 acquireResources n pool = Codensity $ \f -> bracket acquire release (f . Set.toList)
  where
  release :: Set.Set a -> m ()

libs/wai-utilities/src/Network/Wai/Utilities/Error.hs

@@ -31,8 +31,6 @@ import Control.Error
 import Data.Aeson hiding (Error)
 import Data.Aeson.Types (Pair)
 import Data.Domain
-import Data.List.NonEmpty (NonEmpty)
-import Data.List.NonEmpty qualified as NE
 import Data.Text.Lazy.Encoding (decodeUtf8)
 import Imports
 import Network.HTTP.Types
@@ -51,24 +49,23 @@ mkError c l m = Error c l m Nothing
 instance Exception Error
 
 data ErrorData = FederationErrorData
- { federrDomains :: NonEmpty Domain,
+ { federrDomain :: !Domain,
  federrPath :: !Text
  }
  deriving (Eq, Show, Typeable)
 
 instance ToJSON ErrorData where
- toJSON (FederationErrorData ds p) =
+ toJSON (FederationErrorData d p) =
  object
  [ "type" .= ("federation" :: Text),
- "domain" .= NE.head ds, -- deprecated in favour for `domains`
- "domains" .= ds,
+ "domain" .= d,
  "path" .= p
  ]
 
 instance FromJSON ErrorData where
  parseJSON = withObject "ErrorData" $ \o ->
  FederationErrorData
- <$> o .: "domains"
+ <$> o .: "domain"
  <*> o .: "path"
 
 -- | Assumes UTF-8 encoding.