Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

When a patch fails, import the Instance directly using GetObjects #786

Draft
wants to merge 34 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
a67aec4
Pass ServeSession to reify
Dekkonot Aug 27, 2023
0017861
Use tempfile as a normal dependency
Dekkonot Aug 28, 2023
70b1899
Store a lazy ref to a temp dir in ServeSession
Dekkonot Aug 28, 2023
ef90926
Initial implementation of `fetch` endpoint
Dekkonot Aug 28, 2023
a5af0e8
Add initial implementation of fetch api for client
Dekkonot Aug 28, 2023
7145d70
Don't actually pass ServeSession to reify
Dekkonot Sep 17, 2023
f2cdcae
Pass apiContext to reconciler and applyPatch
Dekkonot Sep 17, 2023
24ce366
Make `fetchInstances` its own module
Dekkonot Sep 17, 2023
de3c15b
Initial implementation of fetchApi in applyPatch
Dekkonot Sep 17, 2023
9c3d84e
Use a folder in the content directory
Dekkonot Sep 17, 2023
c69ebbe
Move tempfile back to dev dependencies
Dekkonot Sep 17, 2023
2abac70
Handle an invalid request better
Dekkonot Sep 17, 2023
e057711
Rename fetch dir constant
Dekkonot Sep 17, 2023
04b3479
Actually remove the old copy of fetchInstances from reify
Dekkonot Sep 17, 2023
af7e372
Account for classes with specific parent needs
Dekkonot Sep 18, 2023
b8829a0
Add explanatory comment
Dekkonot Sep 18, 2023
e778863
Change how fetchInstances checks invariants
Dekkonot Sep 18, 2023
190d458
Change debug for swapping instances to trace
Dekkonot Sep 18, 2023
47dbf1b
Use request body instead of URL for fetch endpoint
Dekkonot Sep 18, 2023
d77e5f2
Remove unhelpful debug log
Dekkonot Sep 18, 2023
e0c4380
Reference instances by key instead of name
Dekkonot Sep 18, 2023
d905679
Don't clone subtree when building fetch model
Dekkonot Sep 19, 2023
08fd714
Merge branch 'master' into fetch-bad-patch
Dekkonot Sep 19, 2023
e721e6e
StyLua
Dekkonot Sep 19, 2023
2c02328
Preserve selection when replacing instances
Dekkonot Sep 19, 2023
6d6e0fe
Cleanup fetch endpoint
Dekkonot Sep 19, 2023
6666e11
Remove unused imports in Reify
Dekkonot Sep 19, 2023
d7ace0f
Remove extra argument to `applyPatch` in ServeSession
Dekkonot Sep 19, 2023
94cf688
Create .git-blame-ignore-revs & Add Stylua pr (#787)
Barocena Sep 19, 2023
385340f
Add setting to turn functionality off
Dekkonot Sep 22, 2023
81a5c0f
Lock fetchOnPatchFail setting while sync is active
Dekkonot Sep 22, 2023
424b1d0
Move ChangeHistoryService waypoint creation to Reconciler
Dekkonot Sep 22, 2023
a5321be
Move fetchinstances call to reconciler
Dekkonot Sep 22, 2023
d21ffe1
Respect setting for fetch usage in reconciler
Dekkonot Sep 22, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# stylua formatting
0f8e1625d572a5fe0f7b5c08653ff92cc837d346
19 changes: 19 additions & 0 deletions plugin/src/ApiContext.lua
Original file line number Diff line number Diff line change
Expand Up @@ -239,4 +239,23 @@ function ApiContext:open(id)
end)
end

function ApiContext:fetch(ids: { string })
local url = ("%s/api/fetch"):format(self.__baseUrl)
local requestBody = {
sessionId = self.__sessionId,
idList = ids,
}

return Http.post(url, Http.jsonEncode(requestBody))
:andThen(rejectFailedRequests)
:andThen(Http.Response.json)
:andThen(function(responseBody)
if responseBody.sessionId ~= self.__sessionId then
return Promise.reject("Server changed ID")
end

return responseBody
end)
end

return ApiContext
14 changes: 12 additions & 2 deletions plugin/src/App/StatusPages/Settings/init.lua
Original file line number Diff line number Diff line change
Expand Up @@ -155,14 +155,24 @@ function SettingsPage:render()
layoutOrder = 5,
}),

FetchOnPatchFail = e(Setting, {
id = "fetchOnPatchFail",
name = "Load Model On Patch Fail",
description = "Whenever a patch fails to fully apply, send a request to the server to have it "
.. "placed into the local file system for the plugin to load.",
locked = self.props.syncActive,
transparency = self.props.transparency,
layoutOrder = 6,
}),

OpenScriptsExternally = e(Setting, {
id = "openScriptsExternally",
name = "Open Scripts Externally",
description = "Attempt to open scripts in an external editor",
locked = self.props.syncActive,
experimental = true,
transparency = self.props.transparency,
layoutOrder = 6,
layoutOrder = 7,
}),

TwoWaySync = e(Setting, {
Expand All @@ -172,7 +182,7 @@ function SettingsPage:render()
locked = self.props.syncActive,
experimental = true,
transparency = self.props.transparency,
layoutOrder = 7,
layoutOrder = 8,
}),

LogLevel = e(Setting, {
Expand Down
2 changes: 2 additions & 0 deletions plugin/src/App/init.lua
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ function App:startSession()
local sessionOptions = {
openScriptsExternally = Settings:get("openScriptsExternally"),
twoWaySync = Settings:get("twoWaySync"),
fetchOnPatchFail = Settings:get("fetchOnPatchFail"),
}

local baseUrl = if string.find(host, "^https?://")
Expand All @@ -358,6 +359,7 @@ function App:startSession()
apiContext = apiContext,
openScriptsExternally = sessionOptions.openScriptsExternally,
twoWaySync = sessionOptions.twoWaySync,
fetchOnPatchFail = sessionOptions.fetchOnPatchFail,
})

self.cleanupPrecommit = serveSession.__reconciler:hookPrecommit(function(patch, instanceMap)
Expand Down
6 changes: 0 additions & 6 deletions plugin/src/Reconciler/applyPatch.lua
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
Patches can come from the server or be generated by the client.
]]

local ChangeHistoryService = game:GetService("ChangeHistoryService")

local Packages = script.Parent.Parent.Parent.Packages
local Log = require(Packages.Log)

Expand All @@ -19,8 +17,6 @@ local reify = require(script.Parent.reify)
local setProperty = require(script.Parent.setProperty)

local function applyPatch(instanceMap, patch)
local patchTimestamp = DateTime.now():FormatLocalTime("LTS", "en-us")

-- Tracks any portions of the patch that could not be applied to the DOM.
local unappliedPatch = PatchSet.newEmpty()

Expand Down Expand Up @@ -203,8 +199,6 @@ local function applyPatch(instanceMap, patch)
end
end

ChangeHistoryService:SetWaypoint("Rojo: Patch " .. patchTimestamp)

return unappliedPatch
end

Expand Down
73 changes: 73 additions & 0 deletions plugin/src/Reconciler/fetchInstances.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
local Selection = game:GetService("Selection")

local Rojo = script:FindFirstAncestor("Rojo")
local invariant = require(script.Parent.Parent.invariant)

local Log = require(Rojo.Packages.Log)

local function fetchInstances(idList, instanceMap, apiContext)
return apiContext:fetch(idList):andThen(function(body: { sessionId: string, path: string })
-- The endpoint `api/fetech/idlist` returns a table that contains
-- `sessionId` and `path`. The value of `path` is the name of a
-- file in the Content folder that may be loaded via `GetObjects`.
local objects = game:GetObjects("rbxasset://" .. body.path)
-- `ReferentMap` is a folder that contains nothing but
-- ObjectValues which will be named after entries in `instanceMap`
-- and have their `Value` property point towards a new Instance
-- that it can be swapped out with. In turn, `reified` is a
-- container for all of the objects pointed to by those instances.
Comment on lines +14 to +18
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure ReferentMap is necessary - it seems a little complex for what it's actually accomplishing

Because our serializers provide ordering guarantees for children, and GetObject insertion reflects this ordering, I think the server could construct a model file with children in the same order as the requested IDs, and then the client can match them back up via this ordering.

Alternatively, if it's uncomfortable to rely on child order this way, then the server could write multiple files, and respond with multiple paths (again, in the same order as the requested IDs), at the expense of multiple file system writes and multiple calls to GetObjects

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I went with ReferentMap because it was the first solution I thought of that avoided a bunch of FS writes and reads, not necessarily because I thought it was the best option. In practice it's not really all that complicated, it just sounds bad when you write it out. I can get behind wanting to simplify it as much as possible though, since it's unusual for Rojo.

There's the potential to write thousands of files to a file system and then load them in very quickly if we use multiple paths. I'm not inherently opposed to that, but I'd want to check what the performance of that was. It's not great as-is with one monolith model, I can imagine it getting really bad with a bunch of them.

I'm going to veto relying on child ordering just on principle. It makes me feel gross.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I overlooked the additional complication caused by instances with parent restrictions, so ordering wouldn't work anyway... it's probably fine

local map = objects[1]:FindFirstChild("ReferentMap")
local reified = objects[1]:FindFirstChild("Reified")
if map == nil then
invariant("The fetch endpoint returned a malformed folder: missing ReferentMap")
end
if reified == nil then
invariant("The fetch endpoint returned a malformed folder: missing Reified")
end

-- We want to preserve selection between replacements.
local selected = Selection:Get()
local selectedMap = {}
for i, inst in selected do
selectedMap[inst] = i
end

for _, entry in map:GetChildren() do
if entry:IsA("ObjectValue") then
local key, value = entry.Name, entry.Value
if value == nil or not value:IsDescendantOf(reified) then
invariant("ReferentMap contained entry {} that was parented to an outside source", key)
else
-- This could be a problem if Roblox ever supports
-- parallel access to the DataModel but right now,
-- there's no way this results in a data race.
local oldInstance: Instance = instanceMap.fromIds[key]
instanceMap:insert(key, value)
Log.trace("Swapping Instance {} out", key)

local oldParent = oldInstance.Parent
local children = oldInstance:GetChildren()
for _, child in children do
child.Parent = value
end
value.Parent = oldParent
if selectedMap[oldInstance] then
-- Swapping section like this preserves order
-- It might not matter, but it's also basically free
-- So we may as well.
selected[selectedMap[oldInstance]] = value
end

-- So long and thanks for all the fish :-)
oldInstance:Destroy()
end
else
invariant("ReferentMap entry `{}` was a `{}` and not an ObjectValue", entry.Name, entry.ClassName)
end
end

Selection:Set(selected)
end)
end

return fetchInstances
30 changes: 29 additions & 1 deletion plugin/src/Reconciler/init.lua
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,28 @@
This module defines the meat of the Rojo plugin and how it manages tracking
and mutating the Roblox DOM.
]]
local ChangeHistoryService = game:GetService("ChangeHistoryService")

local Packages = script.Parent.Parent.Packages
local Log = require(Packages.Log)

local PatchSet = require(script.Parent.PatchSet)

local fetchInstances = require(script.fetchInstances)
local applyPatch = require(script.applyPatch)
local hydrate = require(script.hydrate)
local diff = require(script.diff)

local Reconciler = {}
Reconciler.__index = Reconciler

function Reconciler.new(instanceMap)
function Reconciler.new(instanceMap, apiContext, fetchOnPatchFail: boolean)
local self = {
-- Tracks all of the instances known by the reconciler by ID.
__instanceMap = instanceMap,
-- An API context for sending requests back to the server
__apiContext = apiContext,
__fetchOnPatchFail = fetchOnPatchFail,
__precommitCallbacks = {},
__postcommitCallbacks = {},
}
Expand Down Expand Up @@ -64,8 +71,29 @@ function Reconciler:applyPatch(patch)
end
end

local patchTimestamp = DateTime.now():FormatLocalTime("LTS", "en-us")

local unappliedPatch = applyPatch(self.__instanceMap, patch)

if self.__fetchOnPatchFail then
-- TODO Is it worth doing this for additions that fail?
-- It seems unlikely that a valid Instance can't be made with `Instance.new`
-- but can be made using GetObjects
if PatchSet.hasUpdates(unappliedPatch) then
local idList = table.create(#unappliedPatch.updated)
for i, entry in unappliedPatch.updated do
idList[i] = entry.id
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should also check here that the unapplied updates are for properties, otherwise we may try to fetch instances for updates (like metadata) that GetObjects won't help with

end
-- TODO this is destructive to any properties that are
-- overwritten by the user but not known to Rojo. We can probably
-- mitigate that by keeping tabs of any instances we need to swap.
fetchInstances(idList, self.__instanceMap, self.__apiContext)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know if it makes sense to be performing HTTP requests in the reconciler. I suppose the entire fetch process could conceptually be seen as part of patch application, but I think I'd rather have the API call happen in ServeSession (maybe by wrapping Reconciler:applyPatch in a new function ServeSession:__applyPatch) to better follow the existing architecture, to make the control flow a little clearer, and to avoid passing an ApiContext plus a setting to the reconciler

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't love it, but I'll just be honest: this was exploratory surgery on the plugin. I didn't really know where to put it, so I went with the reconciler because it was at least sensible.

table.clear(unappliedPatch.updated)
end
end

ChangeHistoryService:SetWaypoint("Rojo: Patch " .. patchTimestamp)

for _, callback in self.__postcommitCallbacks do
local success, err = pcall(callback, patch, self.__instanceMap, unappliedPatch)
if not success then
Expand Down
4 changes: 3 additions & 1 deletion plugin/src/ServeSession.lua
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ local validateServeOptions = t.strictInterface({
apiContext = t.table,
openScriptsExternally = t.boolean,
twoWaySync = t.boolean,
fetchOnPatchFail = t.boolean,
})

function ServeSession.new(options)
Expand All @@ -73,7 +74,7 @@ function ServeSession.new(options)

local instanceMap = InstanceMap.new(onInstanceChanged)
local changeBatcher = ChangeBatcher.new(instanceMap, onChangesFlushed)
local reconciler = Reconciler.new(instanceMap)
local reconciler = Reconciler.new(instanceMap, options.apiContext, options.fetchOnPatchFail)

local connections = {}

Expand All @@ -91,6 +92,7 @@ function ServeSession.new(options)
__apiContext = options.apiContext,
__openScriptsExternally = options.openScriptsExternally,
__twoWaySync = options.twoWaySync,
__fetchOnPatchFail = options.fetchOnPatchFail,
__reconciler = reconciler,
__instanceMap = instanceMap,
__changeBatcher = changeBatcher,
Expand Down
1 change: 1 addition & 0 deletions plugin/src/Settings.lua
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ local defaultSettings = {
openScriptsExternally = false,
twoWaySync = false,
showNotifications = true,
fetchOnPatchFail = true,
syncReminder = true,
confirmationBehavior = "Initial",
largeChangesConfirmationThreshold = 5,
Expand Down
Loading
Loading