wireapp · zebot · Jul 9, 2024 · Jul 9, 2024 · Jul 12, 2024 · Jul 15, 2024
diff --git a/.gitignore b/.gitignore
@@ -60,6 +60,9 @@ stack-dev.yaml
 # HIE db files (e.g. generated for stan)
 *.hie
 
+# dump timings
+*.dump-timings
+
 # generated files under .local
 .local
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,236 @@
+# [2024-10-09] (Chart Release 5.6.0)
+
+## Release notes
+
+
+* To remove phone keys from brig's `user_keys` table an ad hoc data-migration can be run. See PR https://github.com/wireapp/wire-server/pull/4146 which contains the implementation. (#4130)
+
+* Because the `phone` column is deleted from Brig's `user` table in a schema
+  migration, temporarily there might be 5xx errors during deployment if Wire
+  server 5.4.0 was not deployed previously. To avoid these errors, please deploy
+  the Wire server 5.4.0 release first. (#4130)
+
+* With this release it will be possible to invite personal users to teams. In `brig`'s config, `emailSMS.team.tExistingUserInvitationUrl` is required to be set to a value that points to the correct teams/account page.
+  If `emailSMS.team` is not defined at all in the current environment, the value of `externalUrls.teamSettings` (or, if not present, `externalUrls.nginz`) will be used to construct the correct url, and no configuration change is necessary. (#4229)
+
+* charts/wire-server: There is a new config value called `background-worker.config.enableFederation` which defaults to `false`. This must be kept in sync with `tags.federation`. (#4243)
+
+* If you are mapping an email address to the `externalId` field in the
+  scim schema, please check the following list for items that apply to
+  you and recommended steps before/during/after upgrade.
+
+  - **Situation:** the `emails` field of in your scim user records is
+      empty.
+
+    **What you need to do:** change your schema mapping to contain the
+      same address in `externalId` and (as a record with one element) in
+      `emails`.
+
+  - **Situation:** the `emails` field of your scim user records is
+      non-empty.
+
+    **What you need to do:** make sure `emails` contains exactly one
+      entry, which is the email from `externalId`.  If there is a
+      discrepancy, the address from `emails` will become the new
+      (unvalidated) address of the user, and the user will receive an
+      email to validate it.  If the email cannot be sent or is ignored
+      by the recipient, the *valid* address will not be changed. (#4221)
+
+* A schema migration drops column 'phone' from Brig's 'team_invitation' table. Previous releases were still reading this column. As there is no Team Settings UI action to enter a phone number, this reading will not miss to read actual phone numbers. Therefore, during deployment this will lead to benign 5xx errors. (#4149)
+
+
+## API changes
+
+
+* A new endpoint `POST /teams/invitations/accept` allows a non-team user to accept an invitation to join a team (#4229)
+
+* Services allowlist are blocked by 409 (mls-services-not-allowed) for teams with default protocol MLS. (#4266)
+
+* The `POST /clients` and `PUT /clients/:cid` endpoints support a new capability "consume-notifications" (#4259)
+
+* All the phone number-based functionality is removed from the client API v6 (#4149)
+
+* The changes to the `capabilities` field of the `Client` structure, introduced in v6, have now been postponed to v7 (#4179)
+
+* Finalise version 6 and introduce new development version 7 (#4179, #4179)
+
+* From API version 7 the `GET /mls/public-key` and `GET  /conversations/one2one/:domain/:uid` endpoints now take a `format` query parameter which can be either `raw` (default, for raw base64-encoded keys) or `jwk` (for JWK keys) (#4216, #4224)
+
+* `GET /conversations/one2one/:domain/:uid` now returns `public_keys` along with the conversation containing all MLS public keys for the backend which will host this conversation (since v6). (#4224)
+
+* Remove the ability to set the TTL of a feature flag. Existing TTLs are still retrieved and returned as before. Note that this only applies to the conferenceCalling feature, as none of the others supported TTL anyway. (#4164)
+
+* Add useSFTForOneToOneCalls as a config option for the Conference Calling feature flag and make its lock status explicit. (#4164)
+
+* Add endpoint to upgrade a personal user to a team owner (#4251)
+
+
+## Features
+
+
+* DB migration for dropping `phone` column from `user` table (#4130)
+
+* A text status field was added to user and user profile (#4155)
+
+* Allow an existing non-team user to migrate to a team (#4229, #4229)
+
+* Makes it impossible for a user to join an MLS conversation while already under legalhold (at least pending)
+
+  This implies two things:
+  1. If a user is under legalhold they cannot ever join an MLS conversation, not even an MLS self conversation.
+  2. A user has to reject to be put under legalhold when they want to join an MLS conversation (ignoring the request to be put under legalhold is not enough). (#4242)
+
+* Clients can declare to be supporting a capability for consuming notifications (#4259)
+
+* New endpoint to revoke an OAuth session (#4213)
+
+* Adds a field which contains a list of all active sessions to each OAuth application in the response of `GET /oauth/applications` (#4211)
+
+* SCIM's emails field is now handled and the external ID is not restricted to being an email anymore (#4221)
+
+* allow subconversations for MLS 1-1 conversations (#4133)
+
+* Deny requests for a legalhold device for users who are part of any MLS conversations (#4245)
+
+* Allow setting of Kubernetes annotations for the `coturn` Service. (#4189)
+
+* Add `initialConfig` setting for the `mls` feature flag (#4262)
+
+* added open telemetry instrumentation for brig, galley, gundeck and cannon (#3901)
+
+* Send confirmation email after adding a personal user to a new team (#4253)
+
+* The SFT and turn usernames returned by `/calls/config/v2` are now deterministically computed from the user ID (#4156)
+
+* Use latest stable RabbitMQ version (`3.13.7`) and Helm chart (`14.6.9`). Please
+  note that this minor RabbitMQ version upgrade (`3.11.x` to `3.13.x`) may need
+  special treatment regarding existing RabbitMQ instances. See
+  https://www.rabbitmq.com/docs/upgrade#rabbitmq-version-upgradability . The major
+  Helm chart version upgrade may (depending on your setup/values) need attention
+  as well: https://github.com/bitnami/charts/tree/main/bitnami/rabbitmq#upgrading (#4227)
+
+
+## Bug fixes and other updates
+
+
+* Fixed API version check. It has now precedence over other checks like e.g. method check. (#4152)
+
+* Fix handling of defaults of `mlsE2EID` feature config (#4233)
+
+* Match cipher suite tag in query parameters against key packages on replacing key packages (#4158)
+
+* Users with SAML-SSO are allowed to delete their email address on the rest api. If they do that, the search indices are not updated correctly, and finding the user by the removed email address is still possible. (#4260)
+
+* Exclude exception message from error response (#4153)
+
+* Return HTTP 400 instead of 500 when property key is not printable ASCII (#4148)
+
+* move cipher suite updates into the commit lock (#4151)
+
+* Fix feature flag default calculation for `mlsMigration` and `enforceFileDownloadLocation` (#4265)
+
+* Allow setting existing properties even if we have max properties (#4148)
+
+* removed spam from nginx (nginz) by using the new style http/2 directive (#3901)
+
+* brig: Make `GET /services/tags` work again (#4250)
+
+
+## Documentation
+
+
+* Restored LegalHold internal API swagger as part of Brig. (#4191)
+
+* Deleted proteus-specific test documentation tags and added some new tags to MLS tests (#4240)
+
+* Re-introduce test case tags for BSI audit (revert #4041) (#4192)
+
+
+## Internal changes
+
+
+* Read sftTokenSecret from secrets.yaml and mount to /etc/wire/brig/secrets/sftTokenSecret by default (#4214)
+
+* Added node based topology constraint to ensure pods are distributed uniformly on all nodes. (#4222)
+
+* Move smallstep-accomp` helm charts to `wireapp/helm-charts` (#4204)
+
+* Remove coturn helm chart. It is moved to `wireapp/coturn`. (#4209)
+
+* Additional test for password reset, port tests to new integration test suite (#4249)
+
+* Remove unused invitation tables from brig. (#4263)
+
+* Improve abstraction in the invitation store and hide DB interaction-specific internal types from the application code. (#4280)
+
+* Move some invitation handling from brig to wire-subsystems.
+
+  - introduce cyclically dependent effects: UserSubsystem, AuthenticationSubsystem (see Brig.CanonicalInterpreter).
+  - introduce TeamInvitationSubsystem with operations inviteUser, internalCreateInvitation.
+  - add verifyPassword to AuthenticationSubsystem.
+  - add sendInvitationMail, sendInvitationMailPersonalUser to EmailSubsystem.
+  - add getTeamSize to IndexedUserStore (this is morally internal to wire-subsystems, and making another ES subsystem would mean adding a lot of code everywhere).
+  - add updateUserTeam to UserStore.
+  - add acceptTeamInvitation, internalFindTeamInvitation to UserSubsystem.
+  - make a few small rest api handlers in brig polysemic (Handler -> Sem). (#4264)
+
+* tools/db/team-info: collects last login times of all team members (#4274)
+
+* Introduce length-preserving function mapRange to replace Functor instance for Range data type. (#4279)
+
+* Servantify gundeck internal api (#4246)
+
+* Removed `indexReindex` and `indexReindexIfSameOrNewer` from internal Brig/SearchIndex. (#4188)
+
+* Introduced ElasticSearch effects related to user search. (#4188)
+
+* Brig was refactored by pulling out email block-listing into a wire subsystems effect, and its actions are exposed via the user subsystem. (#4167)
+
+* charts/wire-server: Deploy background-worker even when tags.federation is `false` (#4342, #4248)
+
+* Refactor feature flags
+  - Improved naming slightly. Features types are now called `Feature`, `LockableFeature` and `LockableFeaturePatch`
+  - Turned `AllFeatures` into an extensible record type
+  - Removed `WithStatusBase` barbie.
+  - Deleted obsolete `computeFeatureConfigForTeamUser`
+  - Abstracted `getFeature` and `setFeature`
+  - Abstracted getAllTeamFeatures (#4181)
+
+* Clean up and reorganise feature flag endpoints (#4193)
+
+* Clean up feature default configuration code (#4196)
+
+* Add federation-v1 environment for testing compatibility of the federation API with version 1 (#4125)
+
+* Expose gundeck internal API on swagger.  Mv some types and routes to wire-api. (#4247)
+
+* Postgresql helm chart is removed from charts/ directory and migrated to wireapp/helm-charts repo (#4208)
+
+* Simplify NewTeam and related types and remove lenses (#4257)
+
+* Optimize getting a lot of users by concurrently getting target users (#4140)
+
+* charts/{brig,galley}: Allow setting a preStop hook for the deployments (#4200)
+
+* Introduce proeprty subsytem (#4148)
+
+* Changed default password hashing from Scrypt to Argon2id. (#4271)
+
+* Factored out our Email type in favour of EmailAddress from email-validate. (#4206)
+
+* add the TODO pattern and the todo function to Imports (#4198)
+
+* Refactor user feature logic (#4178)
+
+* Remove `UserAccount` and `ExtendedUserAccount` and their fields to the `User` type (#4275)
+
+* Started weeding out dead code. (#4170)
+
+* New user subsystem operation `getAccountsBy` for complex account lookups. (#4218)
+
+* Added warning when deploying wire-server helm chart with User/Team creation over internet enabled. (#4212)
+
+
 # [2024-07-09] (Chart Release 5.5.0)
 
 ## Bug fixes and other updates

diff --git a/Makefile b/Makefile
@@ -7,7 +7,7 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana restund coturn k8ssandra-test-cluster
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana restund k8ssandra-test-cluster
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
@@ -17,8 +17,8 @@ CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster rabbitmq rabbitmq-ex
 fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
-nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper restund coturn		\
-k8ssandra-test-cluster postgresql ldap-scim-bridge smallstep-accomp
+nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper restund \
+k8ssandra-test-cluster ldap-scim-bridge
 KIND_CLUSTER_NAME     := wire-server
 HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 
@@ -49,12 +49,17 @@ install: init
 	./hack/bin/cabal-run-all-tests.sh
 	./hack/bin/cabal-install-artefacts.sh all
 
+.PHONY: clean-rabbit
+clean-rabbit:
+	rabbitmqadmin -f pretty_json list queues vhost name messages | jq -r '.[] | "rabbitmqadmin delete queue name=\(.name) --vhost=\(.vhost)"' | bash
+
 # Clean
 .PHONY: full-clean
 full-clean: clean
 	rm -rf ~/.cache/hie-bios
 	rm -rf ./dist-newstyle ./.env
 	direnv reload
+	clean-rabbit
 	@echo -e "\n\n*** NOTE: you may want to also 'rm -rf ~/.cabal/store \$$CABAL_DIR/store', not sure.\n"
 
 .PHONY: clean
@@ -77,7 +82,10 @@ cabal.project.local:
 
 # Usage: make c package=brig test=1
 .PHONY: c
-c: treefmt
+c: treefmt c-fast
+
+.PHONY: c
+c-fast: 
 	cabal build $(WIRE_CABAL_BUILD_OPTIONS) $(package) || ( make clean-hint; false )
 ifeq ($(test), 1)
 	./hack/bin/cabal-run-tests.sh $(package) $(testargs)
@@ -162,7 +170,7 @@ lint-all: formatc hlint-check-all lint-common
 # The extra 'hlint-check-pr' has been witnessed to be necessary due to
 # some bu in `hlint-inplace-pr`.  Details got lost in history.
 .PHONY: lint-all-shallow
-lint-all-shallow: formatf hlint-inplace-pr hlint-check-pr lint-common
+lint-all-shallow: lint-common formatf hlint-inplace-pr hlint-check-pr 
 
 .PHONY: lint-common
 lint-common: check-local-nix-derivations treefmt-check # weeder (does not work on CI yet)
@@ -238,11 +246,11 @@ add-license:
 
 .PHONY: treefmt
 treefmt:
-	treefmt
+	treefmt -u debug
 
 .PHONY: treefmt-check
 treefmt-check:
-	treefmt --fail-on-change
+	treefmt --fail-on-change -u debug
 
 #################################
 ## docker targets

diff --git a/cabal.project b/cabal.project
@@ -11,7 +11,6 @@ packages:
   , libs/dns-util/
   , libs/deriving-swagger2/
   , libs/galley-types/
-  , libs/gundeck-types/
   , libs/hscim/
   , libs/http2-manager/
   , libs/imports/
@@ -29,6 +28,7 @@ packages:
   , libs/wai-utilities/
   , libs/wire-api/
   , libs/wire-api-federation/
+  , libs/wire-otel/
   , libs/wire-message-proto-lens/
   , libs/wire-subsystems/
   , libs/zauth/
@@ -49,6 +49,7 @@ packages:
   , tools/db/move-team/
   , tools/db/phone-users/
   , tools/db/repair-handles/
+  , tools/db/team-info/
   , tools/db/repair-brig-clients-table/
   , tools/db/service-backfill/
   , tools/fedcalls/
@@ -74,3 +75,4 @@ program-options
 allow-newer:
   , proto-lens-protoc:base
   , proto-lens-protoc:ghc
+  , proto-lens-setup:Cabal