Merge branch 'main' into fp/quickjournal-sync

* main:
  Add new subscribe API (#3403)
  Update QuickJournal (#3401)
  Add a few tests for uncovered code (#3405)
  Update to Core 13.17.1 (#3404)
  Use modern-er marshaling techniques (#3261)

.editorconfig

@@ -19,7 +19,7 @@ insert_final_newline = true
 # Organize usings
 dotnet_separate_import_directive_groups = false
 dotnet_sort_system_directives_first = true
-file_header_template = ////////////////////////////////////////////////////////////////////////////\n//\n// Copyright 2022 Realm Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the "License")\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an "AS IS" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n////////////////////////////////////////////////////////////////////////////
+file_header_template = ////////////////////////////////////////////////////////////////////////////\n//\n// Copyright 2023 Realm Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the "License")\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an "AS IS" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n////////////////////////////////////////////////////////////////////////////
 
 # this. and Me. preferences
 dotnet_style_qualification_for_event = false:silent
@@ -182,26 +182,26 @@ dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case
 
 dotnet_naming_symbols.interface.applicable_kinds = interface
 dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
-dotnet_naming_symbols.interface.required_modifiers = 
+dotnet_naming_symbols.interface.required_modifiers =
 
 dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum
 dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
-dotnet_naming_symbols.types.required_modifiers = 
+dotnet_naming_symbols.types.required_modifiers =
 
 dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method
 dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
-dotnet_naming_symbols.non_field_members.required_modifiers = 
+dotnet_naming_symbols.non_field_members.required_modifiers =
 
 # Naming styles
 
-dotnet_naming_style.pascal_case.required_prefix = 
-dotnet_naming_style.pascal_case.required_suffix = 
-dotnet_naming_style.pascal_case.word_separator = 
+dotnet_naming_style.pascal_case.required_prefix =
+dotnet_naming_style.pascal_case.required_suffix =
+dotnet_naming_style.pascal_case.word_separator =
 dotnet_naming_style.pascal_case.capitalization = pascal_case
 
 dotnet_naming_style.begins_with_i.required_prefix = I
-dotnet_naming_style.begins_with_i.required_suffix = 
-dotnet_naming_style.begins_with_i.word_separator = 
+dotnet_naming_style.begins_with_i.required_suffix =
+dotnet_naming_style.begins_with_i.word_separator =
 dotnet_naming_style.begins_with_i.capitalization = pascal_case
 
 # Rule Sets

CHANGELOG.md

@@ -1,17 +1,35 @@
 ## vNext (TBD)
 
 ### Enhancements
-* None
+* Added `IQueryable.SubscribeAsync` API as a shorthand for using `SubscriptionSet.Add`. It is a syntax sugar that roughly translates to:
+ ```csharp
+ realm.Subscriptions.Update(() =>
+ {
+ realm.Subscriptions.Add(query);
+ });
+
+ await realm.Subscriptions.WaitForSynchronization();
+
+ // This can now be expressed as
+ await query.SubscribeAsync();
+ ```
+ It offers a parameter to control whether to wait every time for synchronization or just the first time a subscription is added, as well as cancellation token support. (PR [#3403](https://github.com/realm/realm-dotnet/pull/3403))
+* Added an optional `cancellationToken` argument to `Session.WaitForDownloadAsync/WaitForUploadAsync`. (PR [#3403](https://github.com/realm/realm-dotnet/pull/3403))
+* Added an optional `cancellationToken` argument to `SubscriptionSet.WaitForSynchronization`. (PR [#3403](https://github.com/realm/realm-dotnet/pull/3403))
+* Fixed a rare corruption of files on streaming format (often following compact, convert or copying to a new file). (Core 13.17.1)
+* Trying to search a full-text indexes created as a result of an additive schema change (i.e. applying the differences between the local schema and a synchronized realm's schema) could have resulted in an IllegalOperation error with the error code `Column has no fulltext index`. (Core 13.17.1)
+* Sync progress for DOWNLOAD messages from server state was updated wrongly. This may have resulted in an extra round-trip to the server. (Core 13.17.1)
 
 ### Fixed
 * Fixed a race condition between canceling an async write transaction and closing the Realm file, which could result in an `ObjectDisposedException : Safe handle has been closed` being thrown. ([PR #3400](https://github.com/realm/realm-dotnet/pull/3400))
 * Fixed an issue where in the extremely rare case that an exception is thrown by `Realm.RefreshAsync`, that exception would have been ignored and `false` would have been returned. ([PR #3400](https://github.com/realm/realm-dotnet/pull/3400))
+* Fixed the nullability annotation of `SubscriptionSet.Find` to correctly indicate that `null` is returned if the subscription doesn't exist in the subscription set. (PR [#3403](https://github.com/realm/realm-dotnet/pull/3403))
 
 ### Compatibility
 * Realm Studio: 13.0.0 or later.
 
 ### Internal
-* Using Core 13.17.0
+* Using Core 13.17.1
 
 ## 11.3.0 (2023-07-26)
 

Realm/Realm/Configurations/InMemoryConfiguration.cs

@@ -50,16 +50,16 @@ public InMemoryConfiguration(string identifier) : base(identifier)
  Identifier = identifier;
  }
 
- internal override SharedRealmHandle CreateHandle(RealmSchema schema) => SharedRealmHandle.Open(CreateNativeConfiguration(), schema, encryptionKey: null);
+ internal override SharedRealmHandle CreateHandle(in Configuration configuration) => SharedRealmHandle.Open(configuration);
 
- internal override Configuration CreateNativeConfiguration()
+ internal override Configuration CreateNativeConfiguration(Arena arena)
  {
- var result = base.CreateNativeConfiguration();
+ var result = base.CreateNativeConfiguration(arena);
  result.in_memory = true;
  return result;
  }
 
- internal override Task<SharedRealmHandle> CreateHandleAsync(RealmSchema schema, CancellationToken cancellationToken)
- => Task.FromResult(CreateHandle(schema));
+ internal override Task<SharedRealmHandle> CreateHandleAsync(in Configuration configuration, CancellationToken cancellationToken)
+ => Task.FromResult(CreateHandle(configuration));
  }
 }

Realm/Realm/Configurations/RealmConfiguration.cs

@@ -16,6 +16,7 @@
 //
 ////////////////////////////////////////////////////////////////////////////
 
+using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Realms.Helpers;
@@ -139,12 +140,12 @@ public RealmConfiguration ConfigWithPath(string newConfigPath)
  return ret;
  }
 
- internal override SharedRealmHandle CreateHandle(RealmSchema schema)
- => SharedRealmHandle.Open(CreateNativeConfiguration(), schema, EncryptionKey);
+ internal override SharedRealmHandle CreateHandle(in Configuration configuration)
+ => SharedRealmHandle.Open(configuration);
 
- internal override Configuration CreateNativeConfiguration()
+ internal override Configuration CreateNativeConfiguration(Arena arena)
  {
- var result = base.CreateNativeConfiguration();
+ var result = base.CreateNativeConfiguration(arena);
 
  result.delete_if_migration_needed = ShouldDeleteIfMigrationNeeded;
  result.read_only = IsReadOnly;
@@ -155,21 +156,28 @@ internal override Configuration CreateNativeConfiguration()
  return result;
  }
 
- internal override Task<SharedRealmHandle> CreateHandleAsync(RealmSchema schema, CancellationToken cancellationToken)
+ internal override Task<SharedRealmHandle> CreateHandleAsync(in Configuration configuration, CancellationToken cancellationToken)
  {
  // Can't use async/await due to mono inliner bugs
  // If we are on UI thread will be set but often also set on long-lived workers to use Post back to UI thread.
  if (AsyncHelper.TryGetScheduler(out var scheduler))
  {
+ // make a local copy because we can't capture in/ref/out parameters in lambdas
+ var config = configuration;
  return Task.Run(() =>
  {
- using (CreateHandle(schema))
+ // make another copy so we can open a temporary realm on this worker thread
+ // to execute any migrations on the background
+ // without clobbering the original config's managed handle
+ var configCopy = config;
+ configCopy.managed_config = GCHandle.ToIntPtr(GCHandle.Alloc(this));
+ using (CreateHandle(configCopy))
  {
  }
- }, cancellationToken).ContinueWith(_ => CreateHandle(schema), scheduler);
+ }, cancellationToken).ContinueWith(_ => CreateHandle(config), scheduler);
  }
 
- return Task.FromResult(CreateHandle(schema));
+ return Task.FromResult(CreateHandle(configuration));
  }
  }
 }

Realm/Realm/Configurations/RealmConfigurationBase.cs

@@ -22,6 +22,7 @@
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
+using Realms.Native;
 using Realms.Schema;
 
 namespace Realms
@@ -199,25 +200,29 @@ internal RealmConfigurationBase Clone()
  internal virtual Realm CreateRealm()
  {
  var schema = Schema;
- var sharedRealmHandle = CreateHandle(schema);
+ using var arena = new Arena();
+ var sharedRealmHandle = CreateHandle(CreateNativeConfiguration(arena));
  return GetRealm(sharedRealmHandle, schema);
  }
 
  internal virtual async Task<Realm> CreateRealmAsync(CancellationToken cancellationToken)
  {
  var schema = Schema;
- var sharedRealmHandle = await CreateHandleAsync(schema, cancellationToken);
+ using var arena = new Arena();
+ var configuration = CreateNativeConfiguration(arena);
+ var sharedRealmHandle = await CreateHandleAsync(configuration, cancellationToken);
  return GetRealm(sharedRealmHandle, schema);
  }
 
- internal virtual Native.Configuration CreateNativeConfiguration()
+ internal virtual Configuration CreateNativeConfiguration(Arena arena)
  {
  var managedConfig = GCHandle.Alloc(this);
 
- var config = new Native.Configuration
+ var config = new Configuration
  {
- Path = DatabasePath,
- FallbackPipePath = FallbackPipePath,
+ path = StringValue.AllocateFrom(DatabasePath, arena),
+ fallbackPipePath = StringValue.AllocateFrom(FallbackPipePath, arena),
+ schema = Schema.ToNative(arena),
  schema_version = SchemaVersion,
  enable_cache = EnableCache,
  max_number_of_active_versions = MaxNumberOfActiveVersions,
@@ -226,6 +231,7 @@ internal virtual Native.Configuration CreateNativeConfiguration()
 #pragma warning restore CS0618 // Type or member is obsolete
  invoke_initial_data_callback = PopulateInitialData != null,
  managed_config = GCHandle.ToIntPtr(managedConfig),
+ encryption_key = MarshaledVector<byte>.AllocateFrom(EncryptionKey, arena),
  };
 
  return config;
@@ -251,8 +257,8 @@ internal Realm GetRealm(SharedRealmHandle sharedRealmHandle, RealmSchema? schema
  return new Realm(sharedRealmHandle, this, schema);
  }
 
- internal abstract SharedRealmHandle CreateHandle(RealmSchema schema);
+ internal abstract SharedRealmHandle CreateHandle(in Configuration configuration);
 
- internal abstract Task<SharedRealmHandle> CreateHandleAsync(RealmSchema schema, CancellationToken cancellationToken);
+ internal abstract Task<SharedRealmHandle> CreateHandleAsync(in Configuration configuration, CancellationToken cancellationToken);
  }
 }

Realm/Realm/Configurations/SyncConfigurationBase.cs

@@ -21,6 +21,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Realms.Helpers;
+using Realms.Native;
 using Realms.Sync.ErrorHandling;
 using Realms.Sync.Exceptions;
 
@@ -114,19 +115,19 @@ private protected SyncConfigurationBase(User user, string databasePath)
  User = user;
  }
 
- internal override SharedRealmHandle CreateHandle(Schema.RealmSchema schema)
+ internal override SharedRealmHandle CreateHandle(in Configuration configuration)
  {
  var syncConfiguration = CreateNativeSyncConfiguration();
- return SharedRealmHandle.OpenWithSync(CreateNativeConfiguration(), syncConfiguration, schema, EncryptionKey);
+ return SharedRealmHandle.OpenWithSync(configuration, syncConfiguration);
  }
 
- internal override async Task<SharedRealmHandle> CreateHandleAsync(Schema.RealmSchema schema, CancellationToken cancellationToken)
+ internal override Task<SharedRealmHandle> CreateHandleAsync(in Configuration configuration, CancellationToken cancellationToken)
  {
  var syncConfiguration = CreateNativeSyncConfiguration();
 
  var tcs = new TaskCompletionSource<ThreadSafeReferenceHandle>();
  var tcsHandle = GCHandle.Alloc(tcs);
- using var handle = SharedRealmHandle.OpenWithSyncAsync(CreateNativeConfiguration(), syncConfiguration, schema, EncryptionKey, GCHandle.ToIntPtr(tcsHandle));
+ var handle = SharedRealmHandle.OpenWithSyncAsync(configuration, syncConfiguration, GCHandle.ToIntPtr(tcsHandle));
  cancellationToken.Register(() =>
  {
  if (!handle.IsClosed)
@@ -136,17 +137,23 @@ internal override async Task<SharedRealmHandle> CreateHandleAsync(Schema.RealmSc
  }
  });
 
- using var progressToken = OnBeforeRealmOpen(handle);
-
- try
- {
- using var realmReference = await tcs.Task;
- return SharedRealmHandle.ResolveFromReference(realmReference);
- }
- finally
+ async Task<SharedRealmHandle> WaitForAsyncOpenTask()
  {
- tcsHandle.Free();
+ using var progressToken = OnBeforeRealmOpen(handle);
+
+ try
+ {
+ using var realmReference = await tcs.Task;
+ return SharedRealmHandle.ResolveFromReference(realmReference);
+ }
+ finally
+ {
+ tcsHandle.Free();
+ handle.Dispose();
+ }
  }
+
+ return WaitForAsyncOpenTask();
  }
 
  internal virtual IDisposable? OnBeforeRealmOpen(AsyncOpenTaskHandle handle) => null;

Realm/Realm/DatabaseTypes/Accessors/ManagedAccessor.cs

@@ -213,7 +213,7 @@ public void UnsubscribeFromNotifications()
  {
  if (changes.HasValue)
  {
- foreach (var propertyIndex in changes.Value.Properties.AsEnumerable().Select(i => (int)i))
+ foreach (int propertyIndex in changes.Value.Properties)
  {
  // Due to a yet another Mono compiler bug, using LINQ fails here :/
  var i = 0;
@@ -235,7 +235,7 @@ public void UnsubscribeFromNotifications()
  }
  }
 
- if (changes.Value.Deletions.AsEnumerable().Any())
+ if (changes.Value.Deletions.Count > 0)
  {
  RaisePropertyChanged(nameof(IsValid));
 

Realm/Realm/DatabaseTypes/RealmCollectionBase.cs

@@ -404,11 +404,11 @@ private void UpdateCollectionChangedSubscriptionIfNecessary(bool isSubscribed)
  {
  var actualChanges = changes.Value;
  changeset = new ChangeSet(
- insertedIndices: actualChanges.Insertions.AsEnumerable().Select(i => (int)i).ToArray(),
- modifiedIndices: actualChanges.Modifications.AsEnumerable().Select(i => (int)i).ToArray(),
- newModifiedIndices: actualChanges.Modifications_New.AsEnumerable().Select(i => (int)i).ToArray(),
- deletedIndices: actualChanges.Deletions.AsEnumerable().Select(i => (int)i).ToArray(),
- moves: actualChanges.Moves.AsEnumerable().Select(m => new ChangeSet.Move((int)m.From, (int)m.To)).ToArray(),
+ insertedIndices: actualChanges.Insertions.ToEnumerable().Select(i => (int)i).ToArray(),
+ modifiedIndices: actualChanges.Modifications.ToEnumerable().Select(i => (int)i).ToArray(),
+ newModifiedIndices: actualChanges.Modifications_New.ToEnumerable().Select(i => (int)i).ToArray(),
+ deletedIndices: actualChanges.Deletions.ToEnumerable().Select(i => (int)i).ToArray(),
+ moves: actualChanges.Moves.ToEnumerable().Select(m => new ChangeSet.Move((int)m.From, (int)m.To)).ToArray(),
  cleared: actualChanges.Cleared);
  }
 

Realm/Realm/DatabaseTypes/RealmDictionary.cs

@@ -260,9 +260,9 @@ private static string ValidateKey(string key)
  {
  var actualChanges = changes.Value;
  changeset = new DictionaryChangeSet(
- deletedKeys: actualChanges.Deletions.AsEnumerable().Select(v => v.AsString()).ToArray(),
- insertedKeys: actualChanges.Insertions.AsEnumerable().Select(v => v.AsString()).ToArray(),
- modifiedKeys: actualChanges.Modifications.AsEnumerable().Select(v => v.AsString()).ToArray());
+ deletedKeys: actualChanges.Deletions.ToEnumerable().Select(v => v.AsString()).ToArray(),
+ insertedKeys: actualChanges.Insertions.ToEnumerable().Select(v => v.AsString()).ToArray(),
+ modifiedKeys: actualChanges.Modifications.ToEnumerable().Select(v => v.AsString()).ToArray());
  }
  else
  {