From 357d646891d8930cee77a362376aab4ee22d042a Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Sat, 20 Dec 2025 14:42:09 +0100 Subject: [PATCH 01/10] [WIP] Speed up cloud project saves --- .../GDCore/Serialization/BinarySerializer.cpp | 272 ++++++++++++++++++ Core/GDCore/Serialization/BinarySerializer.h | 122 ++++++++ GDevelop.js/CMakeLists.txt | 2 +- GDevelop.js/__tests__/Serializer.js | 140 ++++++--- newIDE/app/scripts/make-service-worker.js | 1 + newIDE/app/src/MainFrame/index.js | 24 +- .../CloudProjectWriter.js | 201 ++++++++++++- .../CloudStorageProvider/index.js | 69 +---- .../LocalProjectWriter.js | 171 ++++++++--- .../LocalFileStorageProvider/index.js | 47 +-- newIDE/app/src/ProjectsStorage/index.js | 13 +- .../app/src/Utils/GDevelopServices/Project.js | 11 +- newIDE/app/src/Utils/Serializer.js | 176 ++++++++++++ newIDE/app/src/Utils/Serializer.worker.js | 104 +++++++ 14 files changed, 1116 insertions(+), 237 deletions(-) create mode 100644 Core/GDCore/Serialization/BinarySerializer.cpp create mode 100644 Core/GDCore/Serialization/BinarySerializer.h create mode 100644 newIDE/app/src/Utils/Serializer.worker.js diff --git a/Core/GDCore/Serialization/BinarySerializer.cpp b/Core/GDCore/Serialization/BinarySerializer.cpp new file mode 100644 index 000000000000..0db21d856523 --- /dev/null +++ b/Core/GDCore/Serialization/BinarySerializer.cpp @@ -0,0 +1,272 @@ +/* + * GDevelop Core + * Copyright 2008-2016 Florian Rival (Florian.Rival@gmail.com). All rights + * reserved. This project is released under the MIT License. + */ + +#include "GDCore/Serialization/BinarySerializer.h" + +#include +#include + +namespace gd { + +using NodeType = BinarySerializer::NodeType; + +void BinarySerializer::SerializeToBinary(const SerializerElement& element, + std::vector& outBuffer) { + // Reserve approximate size (heuristic: 1KB minimum) + outBuffer.clear(); + outBuffer.reserve(1024); + + // Write magic header and version + Write(outBuffer, static_cast(0x47444253)); // "GDBS" magic + Write(outBuffer, static_cast(1)); // Version 1 + + // Serialize the element tree + SerializeElement(element, outBuffer); +} + +void BinarySerializer::SerializeElement(const SerializerElement& element, + std::vector& buffer) { + Write(buffer, NodeType::Element); + + // Serialize value + if (element.IsValueUndefined()) { + Write(buffer, NodeType::ValueUndefined); + } else { + SerializeValue(element.GetValue(), buffer); + } + + // Serialize attributes + const auto& attributes = element.GetAllAttributes(); + Write(buffer, static_cast(attributes.size())); + for (const auto& attr : attributes) { + SerializeString(attr.first, buffer); + SerializeValue(attr.second, buffer); + } + + // Serialize array flags + Write(buffer, element.ConsideredAsArray()); + SerializeString(element.ConsideredAsArrayOf(), buffer); + + // Serialize children + const auto& children = element.GetAllChildren(); + Write(buffer, static_cast(children.size())); + for (const auto& child : children) { + SerializeString(child.first, buffer); // Child name + SerializeElement(*child.second, buffer); // Child element (recursive) + } +} + +void BinarySerializer::SerializeValue(const SerializerValue& value, + std::vector& buffer) { + if (value.IsBoolean()) { + Write(buffer, NodeType::ValueBool); + Write(buffer, value.GetBool()); + } else if (value.IsInt()) { + Write(buffer, NodeType::ValueInt); + Write(buffer, value.GetInt()); + } else if (value.IsDouble()) { + Write(buffer, NodeType::ValueDouble); + Write(buffer, value.GetDouble()); + } else if (value.IsString()) { + Write(buffer, NodeType::ValueString); + SerializeString(value.GetString(), buffer); + } else { + // Shouldn't happen, but handle gracefully + Write(buffer, NodeType::ValueUndefined); + } +} + +void BinarySerializer::SerializeString(const gd::String& str, + std::vector& buffer) { + // Convert to UTF-8 + std::string utf8 = str.ToUTF8(); + + // Write length + data + Write(buffer, static_cast(utf8.size())); + buffer.insert(buffer.end(), utf8.begin(), utf8.end()); +} + +bool BinarySerializer::DeserializeFromBinary(const uint8_t* buffer, + size_t bufferSize, + SerializerElement& outElement) { + const uint8_t* ptr = buffer; + const uint8_t* end = buffer + bufferSize; + + // Read and verify magic header + uint32_t magic; + if (!Read(ptr, end, magic) || magic != 0x47444253) { + return false; // Invalid magic + } + + // Read version + uint32_t version; + if (!Read(ptr, end, version) || version != 1) { + return false; // Unsupported version + } + + // Deserialize element tree + return DeserializeElement(ptr, end, outElement); +} + +bool BinarySerializer::DeserializeElement(const uint8_t*& ptr, + const uint8_t* end, + SerializerElement& element) { + NodeType nodeType; + if (!Read(ptr, end, nodeType) || nodeType != NodeType::Element) { + return false; + } + + // Deserialize value + NodeType valueType; + if (!Read(ptr, end, valueType)) return false; + + if (valueType != NodeType::ValueUndefined) { + SerializerValue value; + if (!DeserializeValue(ptr, end, value, valueType)) return false; + element.SetValue(value); + } + + // Deserialize attributes + uint32_t attrCount; + if (!Read(ptr, end, attrCount)) return false; + + for (uint32_t i = 0; i < attrCount; ++i) { + gd::String attrName; + if (!DeserializeString(ptr, end, attrName)) return false; + + SerializerValue attrValue; + NodeType attrValueType; + if (!Read(ptr, end, attrValueType)) return false; + if (!DeserializeValue(ptr, end, attrValue, attrValueType)) return false; + + // Set attribute based on type + if (attrValue.IsBoolean()) { + element.SetAttribute(attrName, attrValue.GetBool()); + } else if (attrValue.IsInt()) { + element.SetAttribute(attrName, attrValue.GetInt()); + } else if (attrValue.IsDouble()) { + element.SetAttribute(attrName, attrValue.GetDouble()); + } else if (attrValue.IsString()) { + element.SetAttribute(attrName, attrValue.GetString()); + } + } + + // Deserialize array flags + bool isArray; + if (!Read(ptr, end, isArray)) return false; + if (isArray) element.ConsiderAsArray(); + + gd::String arrayOf; + if (!DeserializeString(ptr, end, arrayOf)) return false; + if (!arrayOf.empty()) { + element.ConsiderAsArrayOf(arrayOf); + } + + // Deserialize children + uint32_t childCount; + if (!Read(ptr, end, childCount)) return false; + + for (uint32_t i = 0; i < childCount; ++i) { + gd::String childName; + if (!DeserializeString(ptr, end, childName)) return false; + + SerializerElement& child = element.AddChild(childName); + if (!DeserializeElement(ptr, end, child)) return false; + } + + return true; +} + +bool BinarySerializer::DeserializeValue(const uint8_t*& ptr, + const uint8_t* end, + SerializerValue& value, + NodeType valueType) { + switch (valueType) { + case NodeType::ValueBool: { + bool boolVal; + if (!Read(ptr, end, boolVal)) return false; + value.SetBool(boolVal); + break; + } + case NodeType::ValueInt: { + int intVal; + if (!Read(ptr, end, intVal)) return false; + value.SetInt(intVal); + break; + } + case NodeType::ValueDouble: { + double doubleVal; + if (!Read(ptr, end, doubleVal)) return false; + value.SetDouble(doubleVal); + break; + } + case NodeType::ValueString: { + gd::String strVal; + if (!DeserializeString(ptr, end, strVal)) return false; + value.SetString(strVal); + break; + } + case NodeType::ValueUndefined: + // Value remains undefined + break; + default: + return false; // Unknown value type + } + + return true; +} + +bool BinarySerializer::DeserializeString(const uint8_t*& ptr, + const uint8_t* end, + gd::String& str) { + uint32_t length; + if (!Read(ptr, end, length)) return false; + + if (ptr + length > end) return false; + + std::string utf8(reinterpret_cast(ptr), length); + ptr += length; + + str = gd::String::FromUTF8(utf8); + return true; +} + +#ifdef EMSCRIPTEN +extern "C" { + +EMSCRIPTEN_KEEPALIVE uint8_t* createBinarySnapshot(SerializerElement* element, + size_t* outSize) { + if (!element || !outSize) return nullptr; + + std::vector buffer; + gd::BinarySerializer::SerializeToBinary(*element, buffer); + + *outSize = buffer.size(); + uint8_t* result = (uint8_t*)malloc(buffer.size()); + std::memcpy(result, buffer.data(), buffer.size()); + + return result; +} + +EMSCRIPTEN_KEEPALIVE void freeBinarySnapshot(uint8_t* buffer) { free(buffer); } + +EMSCRIPTEN_KEEPALIVE SerializerElement* deserializeBinarySnapshot( + const uint8_t* buffer, size_t size) { + if (!buffer || size == 0) return nullptr; + + SerializerElement* element = new SerializerElement(); + if (!gd::BinarySerializer::DeserializeFromBinary(buffer, size, *element)) { + delete element; + return nullptr; + } + + return element; +} + +} // extern "C" +#endif + +} // namespace gd diff --git a/Core/GDCore/Serialization/BinarySerializer.h b/Core/GDCore/Serialization/BinarySerializer.h new file mode 100644 index 000000000000..154ade88c97b --- /dev/null +++ b/Core/GDCore/Serialization/BinarySerializer.h @@ -0,0 +1,122 @@ +/* + * GDevelop Core + * Copyright 2008-2016 Florian Rival (Florian.Rival@gmail.com). All rights + * reserved. This project is released under the MIT License. + */ + +#ifndef GDCORE_BINARY_SERIALIZER_H +#define GDCORE_BINARY_SERIALIZER_H + +#include +#include +#include + +#include "GDCore/Serialization/SerializerElement.h" + +namespace gd { + +/** + * \brief Fast binary serialization/deserialization for SerializerElement trees. + * + * This format is optimized for speed and compactness, not for human readability + * or long-term storage. Use it for transferring data between threads. + */ +class GD_CORE_API BinarySerializer { + public: + /** + * \brief Serialize a SerializerElement tree to a binary buffer. + * + * \param element The root element to serialize + * \param outBuffer Output buffer that will contain the binary data + */ + static void SerializeToBinary(const SerializerElement& element, + std::vector& outBuffer); + + /** + * \brief Deserialize a binary buffer back to a SerializerElement tree. + * + * \param buffer The binary data + * \param bufferSize Size of the binary data + * \param outElement The output element (will be populated) + * \return true if successful, false if corrupted data + */ + static bool DeserializeFromBinary(const uint8_t* buffer, + size_t bufferSize, + SerializerElement& outElement); + + enum class NodeType : uint8_t { + Element = 0x01, + ValueUndefined = 0x02, + ValueBool = 0x03, + ValueInt = 0x04, + ValueDouble = 0x05, + ValueString = 0x06 + }; + + private: + // Internal serialization + static void SerializeElement(const SerializerElement& element, + std::vector& buffer); + static void SerializeValue(const SerializerValue& value, + std::vector& buffer); + static void SerializeString(const gd::String& str, + std::vector& buffer); + + // Internal deserialization + static bool DeserializeElement(const uint8_t*& ptr, + const uint8_t* end, + SerializerElement& element); + static bool DeserializeValue(const uint8_t*& ptr, + const uint8_t* end, + SerializerValue& value, + NodeType valueType); + static bool DeserializeString(const uint8_t*& ptr, + const uint8_t* end, + gd::String& str); + + // Helper to write primitive types + template + static void Write(std::vector& buffer, const T& value) { + const uint8_t* bytes = reinterpret_cast(&value); + buffer.insert(buffer.end(), bytes, bytes + sizeof(T)); + } + + // Helper to read primitive types + template + static bool Read(const uint8_t*& ptr, const uint8_t* end, T& value) { + if (ptr + sizeof(T) > end) return false; + std::memcpy(&value, ptr, sizeof(T)); + ptr += sizeof(T); + return true; + } +}; + +// Emscripten bindings +#ifdef EMSCRIPTEN +#include + +extern "C" { +/** + * \brief Create a binary snapshot of a SerializerElement. + * \return Pointer to binary data (caller must free with freeBinarySnapshot) + */ +EMSCRIPTEN_KEEPALIVE uint8_t* createBinarySnapshot(SerializerElement* element, + size_t* outSize); + +/** + * \brief Free a binary snapshot created by createBinarySnapshot. + */ +EMSCRIPTEN_KEEPALIVE void freeBinarySnapshot(uint8_t* buffer); + +/** + * \brief Deserialize binary snapshot back to SerializerElement. + * \return New SerializerElement (caller must delete) + */ +EMSCRIPTEN_KEEPALIVE SerializerElement* deserializeBinarySnapshot( + const uint8_t* buffer, size_t size); +} +#endif + +} // namespace gd + +#endif // GDCORE_BINARY_SERIALIZER_H diff --git a/GDevelop.js/CMakeLists.txt b/GDevelop.js/CMakeLists.txt index 4ace410de61e..47fb0df089f9 100644 --- a/GDevelop.js/CMakeLists.txt +++ b/GDevelop.js/CMakeLists.txt @@ -91,7 +91,7 @@ target_link_libraries(GD "-s TOTAL_MEMORY=48MB") # Get some initial memory size target_link_libraries(GD "-s ALLOW_MEMORY_GROWTH=1") target_link_libraries(GD "-s NODEJS_CATCH_EXIT=0") # Don't print the entire GDCore code on error when running in node target_link_libraries(GD "-s ERROR_ON_UNDEFINED_SYMBOLS=0") -target_link_libraries(GD "-s \"EXPORTED_FUNCTIONS=['_free']\"") +target_link_libraries(GD "-s \"EXPORTED_FUNCTIONS=['_free','_malloc','_createBinarySnapshot','_freeBinarySnapshot','_deserializeBinarySnapshot']\"") if("${GDEVELOPJS_BUILD_VARIANT}" STREQUAL "dev") # Disable optimizations at linking time for slightly faster builds. diff --git a/GDevelop.js/__tests__/Serializer.js b/GDevelop.js/__tests__/Serializer.js index 1a4495503476..deef304218bd 100644 --- a/GDevelop.js/__tests__/Serializer.js +++ b/GDevelop.js/__tests__/Serializer.js @@ -1,13 +1,13 @@ const initializeGDevelopJs = require('../../Binaries/embuild/GDevelop.js/libGD.js'); -describe('libGD.js object serialization', function() { +describe('libGD.js object serialization', function () { let gd = null; beforeAll(async () => { gd = await initializeGDevelopJs(); }); - describe('gd.SerializerElement', function() { - it('should support operations on its value', function() { + describe('gd.SerializerElement', function () { + it('should support operations on its value', function () { var element = new gd.SerializerElement(); element.setStringValue('aaa'); expect(element.getStringValue()).toBe('aaa'); @@ -18,7 +18,7 @@ describe('libGD.js object serialization', function() { element.setDoubleValue(123.457); expect(element.getDoubleValue()).toBeCloseTo(123.457); }); - it('should cast values from a type to another', function() { + it('should cast values from a type to another', function () { var element = new gd.SerializerElement(); element.setStringValue('123'); expect(element.getStringValue()).toBe('123'); @@ -30,7 +30,7 @@ describe('libGD.js object serialization', function() { element.setBoolValue(false); expect(element.getBoolValue()).toBe(false); }); - it('should support operations on its children', function() { + it('should support operations on its children', function () { var element = new gd.SerializerElement(); expect(element.hasChild('Missing')).toBe(false); @@ -44,16 +44,13 @@ describe('libGD.js object serialization', function() { element.setChild('Child2', child2); expect( - element - .getChild('Child2') - .getChild('subChild') - .getStringValue() + element.getChild('Child2').getChild('subChild').getStringValue() ).toBe('Hello world!'); }); }); - describe('gd.Serializer', function() { - it('should serialize a Text Object', function() { + describe('gd.Serializer', function () { + it('should serialize a Text Object', function () { var obj = new gd.TextObject('testObject'); obj.setText('Text of the object, with 官话 characters'); @@ -69,31 +66,25 @@ describe('libGD.js object serialization', function() { }); }); - describe('gd.Serializer.fromJSON and gd.Serializer.toJSON', function() { + describe('gd.Serializer.fromJSON and gd.Serializer.toJSON', function () { const checkJsonParseAndStringify = (json) => { const element = gd.Serializer.fromJSON(json); const outputJson = gd.Serializer.toJSON(element); expect(outputJson).toBe(json); - } + }; - it('should unserialize and reserialize JSON (string)', function() { - checkJsonParseAndStringify( - '"a"' - ); - checkJsonParseAndStringify( - '"String with 官话 characters"' - ); - checkJsonParseAndStringify( - '""' - ); + it('should unserialize and reserialize JSON (string)', function () { + checkJsonParseAndStringify('"a"'); + checkJsonParseAndStringify('"String with 官话 characters"'); + checkJsonParseAndStringify('""'); }); - it('should unserialize and reserialize JSON (objects)', function() { + it('should unserialize and reserialize JSON (objects)', function () { checkJsonParseAndStringify( '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3.0},"things":{"0":{"name":"layout0","referenceTo":"/layouts/layout"},"1":{"name":"layout1","referenceTo":"/layouts/layout"},"2":{"name":"layout2","referenceTo":"/layouts/layout"},"3":{"name":"layout3","referenceTo":"/layouts/layout"},"4":{"name":"layout4","referenceTo":"/layouts/layout"}}}' ); }); - it('should unserialize and reserialize JSON (arrays)', function() { + it('should unserialize and reserialize JSON (arrays)', function () { checkJsonParseAndStringify('[]'); checkJsonParseAndStringify('[1]'); checkJsonParseAndStringify('[1,2]'); @@ -106,32 +97,26 @@ describe('libGD.js object serialization', function() { // TODO: test failures - describe('gd.Serializer.fromJSObject and gd.Serializer.toJSObject', function() { + describe('gd.Serializer.fromJSObject and gd.Serializer.toJSObject', function () { const checkJsonParseAndStringify = (json) => { const object = JSON.parse(json); const element = gd.Serializer.fromJSObject(object); const outputObject = gd.Serializer.toJSObject(element); expect(JSON.stringify(outputObject)).toBe(json); - } + }; - it('should unserialize and reserialize JSON (string)', function() { - checkJsonParseAndStringify( - '"a"' - ); - checkJsonParseAndStringify( - '"String with 官话 characters"' - ); - checkJsonParseAndStringify( - '""' - ); + it('should unserialize and reserialize JSON (string)', function () { + checkJsonParseAndStringify('"a"'); + checkJsonParseAndStringify('"String with 官话 characters"'); + checkJsonParseAndStringify('""'); }); - it('should unserialize and reserialize JSON (objects)', function() { + it('should unserialize and reserialize JSON (objects)', function () { checkJsonParseAndStringify( '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3},"things":{"0":{"name":"layout0","referenceTo":"/layouts/layout"},"1":{"name":"layout1","referenceTo":"/layouts/layout"},"2":{"name":"layout2","referenceTo":"/layouts/layout"},"3":{"name":"layout3","referenceTo":"/layouts/layout"},"4":{"name":"layout4","referenceTo":"/layouts/layout"}}}' ); }); - it('should unserialize and reserialize JSON (arrays)', function() { + it('should unserialize and reserialize JSON (arrays)', function () { checkJsonParseAndStringify('[]'); checkJsonParseAndStringify('[1]'); checkJsonParseAndStringify('[1,2]'); @@ -141,4 +126,81 @@ describe('libGD.js object serialization', function() { checkJsonParseAndStringify('{"7":[],"a":[1,2,{"b":3},{"c":[4,5]},6]}'); }); }); + + // describe('gd.createBinarySnapshot and gd.deserializeBinarySnapshot', function () { + // const checkBinaryRoundTrip = (json) => { + // // Create a SerializerElement from JSON + // const originalElement = gd.Serializer.fromJSON(json); + + // // Create binary snapshot + // const binaryBuffer = gd.createBinarySnapshot(originalElement); + // expect(binaryBuffer).toBeInstanceOf(Uint8Array); + // expect(binaryBuffer.length).toBeGreaterThan(0); + + // // Deserialize binary snapshot + // const restoredElement = gd.deserializeBinarySnapshot(binaryBuffer); + + // // Convert back to JSON and compare + // const outputJson = gd.Serializer.toJSON(restoredElement); + + // restoredElement.delete(); + // originalElement.delete(); + + // expect(outputJson).toBe(json); + // }; + + // it('should round-trip simple values', function () { + // checkBinaryRoundTrip('"hello"'); + // checkBinaryRoundTrip('123'); + // checkBinaryRoundTrip('123.456'); + // checkBinaryRoundTrip('true'); + // checkBinaryRoundTrip('false'); + // }); + + // it('should round-trip strings with unicode characters', function () { + // checkBinaryRoundTrip('"String with 官话 characters"'); + // checkBinaryRoundTrip('"Émojis: 🎮🎲🎯"'); + // }); + + // it('should round-trip objects', function () { + // checkBinaryRoundTrip('{}'); + // checkBinaryRoundTrip('{"a":"b"}'); + // checkBinaryRoundTrip('{"a":{"nested":"value"}}'); + // checkBinaryRoundTrip( + // '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3.0}}' + // ); + // }); + + // it('should round-trip arrays', function () { + // checkBinaryRoundTrip('[]'); + // checkBinaryRoundTrip('[1]'); + // checkBinaryRoundTrip('[1,2,3]'); + // checkBinaryRoundTrip('[{"a":1},{"b":2}]'); + // checkBinaryRoundTrip('{"items":[1,2,3],"nested":[{"x":1},{"y":2}]}'); + // }); + + // it('should round-trip a complex object like a Text Object', function () { + // var obj = new gd.TextObject('testObject'); + // obj.setText('Text with 官话 characters'); + + // var serializedElement = new gd.SerializerElement(); + // obj.serializeTo(serializedElement); + + // // Create binary snapshot + // const binaryBuffer = gd.createBinarySnapshot(serializedElement); + + // // Deserialize binary snapshot + // const restoredElement = gd.deserializeBinarySnapshot(binaryBuffer); + + // // Compare JSON output + // const originalJson = gd.Serializer.toJSON(serializedElement); + // const restoredJson = gd.Serializer.toJSON(restoredElement); + + // expect(restoredJson).toBe(originalJson); + + // restoredElement.delete(); + // serializedElement.delete(); + // obj.delete(); + // }); + // }); }); diff --git a/newIDE/app/scripts/make-service-worker.js b/newIDE/app/scripts/make-service-worker.js index 33569ded1884..8b4d19bde320 100644 --- a/newIDE/app/scripts/make-service-worker.js +++ b/newIDE/app/scripts/make-service-worker.js @@ -56,6 +56,7 @@ const buildSW = () => { 'static/js/locales-*.js', // Locales. 'static/js/!local-app*.js', // Exclude electron app. 'static/js/Resource3DPreview.worker.*.js', // Include the 3D preview worker + 'static/js/Serializer.worker.*.js', // Include the serializer worker // ...But not libGD.js/wasm (there are cached with their URL // query string that depends on the VersionMetadata, see below). diff --git a/newIDE/app/src/MainFrame/index.js b/newIDE/app/src/MainFrame/index.js index 48ed7957612a..8598fb7f2ccb 100644 --- a/newIDE/app/src/MainFrame/index.js +++ b/newIDE/app/src/MainFrame/index.js @@ -3907,6 +3907,7 @@ const MainFrame = (props: Props) => { const saveProject = React.useCallback( async () => { + console.log('saveProject started'); if (!currentProject) return; // Prevent saving if there are errors in the extension modules, as // this can lead to corrupted projects. @@ -3923,16 +3924,19 @@ const MainFrame = (props: Props) => { return; } + console.log('getStorageProviderOperations call'); const storageProviderOperations = getStorageProviderOperations(); const { onSaveProject, - canFileMetadataBeSafelySaved, + // canFileMetadataBeSafelySaved, } = storageProviderOperations; if (!onSaveProject) { return saveProjectAs(); } + console.log('save ui settings'); saveUiSettings(state.editorTabs); + console.log('save ui settings done'); // Protect against concurrent saves, which can trigger issues with the // file system. @@ -3947,16 +3951,8 @@ const MainFrame = (props: Props) => { message: t`You're trying to save changes made to a previous version of your project. If you continue, it will be used as the new latest version.`, }); if (!shouldRestoreCheckedOutVersion) return; - } else if (canFileMetadataBeSafelySaved) { - const canProjectBeSafelySaved = await canFileMetadataBeSafelySaved( - currentFileMetadata, - { - showAlert, - showConfirmation, - } - ); - if (!canProjectBeSafelySaved) return; } + console.log('show message'); _showSnackMessage(i18n._(t`Saving...`), null); setIsSavingProject(true); @@ -3972,6 +3968,8 @@ const MainFrame = (props: Props) => { const saveOptions = {}; if (cloudProjectRecoveryOpenedVersionId) { saveOptions.previousVersion = cloudProjectRecoveryOpenedVersionId; + } else { + saveOptions.previousVersion = currentFileMetadata.version; } if (checkedOutVersionStatus) { saveOptions.restoredFromVersionId = @@ -3980,7 +3978,11 @@ const MainFrame = (props: Props) => { const { wasSaved, fileMetadata } = await onSaveProject( currentProject, currentFileMetadata, - saveOptions + saveOptions, + { + showAlert, + showConfirmation, + } ); if (wasSaved) { diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js index 9e99a81deddb..30657e6e2b99 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js @@ -7,11 +7,15 @@ import { commitVersion, createCloudProject, getCredentialsForCloudProject, + getPresignedUrlForVersionUpload, updateCloudProject, } from '../../Utils/GDevelopServices/Project'; import type { $AxiosError } from 'axios'; import type { MessageDescriptor } from '../../Utils/i18n/MessageDescriptor.flow'; -import { serializeToJSON } from '../../Utils/Serializer'; +import { + serializeToJSObjectInBackground, + serializeToJSON, +} from '../../Utils/Serializer'; import { t } from '@lingui/macro'; import { createZipWithSingleTextFile, @@ -22,13 +26,49 @@ import { getProjectCache } from './CloudProjectOpener'; import { retryIfFailed } from '../../Utils/RetryIfFailed'; import { extractGDevelopApiErrorStatusAndCode } from '../../Utils/GDevelopServices/Errors'; import SaveAsOptionsDialog from '../SaveAsOptionsDialog'; +import { + type ShowAlertFunction, + type ShowConfirmFunction, +} from '../../UI/Alert/AlertContext'; +import { + getCloudProject, + type CloudProjectWithUserAccessInfo, +} from '../../Utils/GDevelopServices/Project'; +import { format } from 'date-fns'; +import { getUserPublicProfile } from '../../Utils/GDevelopServices/User'; const zipProject = async (project: gdProject): Promise<[Blob, string]> => { + console.log('--- zipProject started'); + const startTime = Date.now(); const projectJson = serializeToJSON(project); + const serializeToJSONTime = Date.now(); + console.log( + '--- serializeToJSON done in: ', + serializeToJSONTime - startTime, + 'ms' + ); + + // const startTime2 = Date.now(); + // // TODO: should serialize to JSON instead of JS object. + // const serializedProjectObject = await serializeToJSObjectInBackground( + // project + // ); + // console.log( + // '--- serializeToJSObjectInBackground done in: ', + // Date.now() - startTime2, + // 'ms (in total, including worker promise)' + // ); + + const zipStartTime = Date.now(); const zippedProject = await createZipWithSingleTextFile( projectJson, 'game.json' ); + console.log( + '[Main thread] Zipping done in: ', + Date.now() - zipStartTime, + 'ms' + ); return [zippedProject, projectJson]; }; @@ -47,7 +87,7 @@ const checkZipContent = async ( } }; -const zipProjectAndCommitVersion = async ({ +const zipAndPrepareProjectVersionForCommit = async ({ authenticatedUser, project, cloudProjectId, @@ -57,18 +97,47 @@ const zipProjectAndCommitVersion = async ({ project: gdProject, cloudProjectId: string, options?: {| previousVersion?: string, restoredFromVersionId?: string |}, -|}): Promise => { - const [zippedProject, projectJson] = await zipProject(project); +|}): Promise<{| + presignedUrl: string, + zippedProject: Blob, +|}> => { + const [presignedUrl, [zippedProject, projectJson]] = await Promise.all([ + getPresignedUrlForVersionUpload(authenticatedUser, cloudProjectId), + zipProject(project), + ]); + const archiveIsSane = await checkZipContent(zippedProject, projectJson); if (!archiveIsSane) { throw new Error('Project compression failed before saving the project.'); } + if (!presignedUrl) { + throw new Error( + 'No presigned url was returned from getting presigned url for version upload api call.' + ); + } + + return { presignedUrl, zippedProject }; +}; +const commitProjectVersion = async ({ + authenticatedUser, + presignedUrl, + zippedProject, + cloudProjectId, + options, +}: {| + authenticatedUser: AuthenticatedUser, + presignedUrl: string, + zippedProject: Blob, + cloudProjectId: string, + options?: {| previousVersion?: string, restoredFromVersionId?: string |}, +|}): Promise => { const newVersion = await retryIfFailed({ times: 2 }, () => commitVersion({ authenticatedUser, cloudProjectId, zippedProject, + presignedUrl, previousVersion: options ? options.previousVersion : null, restoredFromVersionId: options ? options.restoredFromVersionId : null, }) @@ -81,13 +150,18 @@ export const generateOnSaveProject = ( ) => async ( project: gdProject, fileMetadata: FileMetadata, - options?: {| previousVersion?: string, restoredFromVersionId?: string |} + options?: {| previousVersion?: string, restoredFromVersionId?: string |}, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} ) => { const cloudProjectId = fileMetadata.fileIdentifier; const gameId = project.getProjectUuid(); const now = Date.now(); if (!fileMetadata.gameId) { + // This is a rare case so we don't do it in parallel with the rest. console.info('Game id was never set, updating the cloud project.'); try { await updateCloudProject(authenticatedUser, cloudProjectId, { @@ -98,9 +172,30 @@ export const generateOnSaveProject = ( // Do not throw, as this is not a blocking error. } } - const newVersion = await zipProjectAndCommitVersion({ + + // Do as much as possible in parallel: + const [canBeSafelySaved, { presignedUrl, zippedProject }] = await Promise.all( + [ + // Check (with a network call) if the project can be safely saved (not modified by someone else). + canFileMetadataBeSafelySaved(authenticatedUser, fileMetadata, actions), + // At the same time, serialize & zip the project and also get (with a network call) a presigned url to upload the project version. + zipAndPrepareProjectVersionForCommit({ + authenticatedUser, + project, + cloudProjectId, + options, + }), + ] + ); + if (!canBeSafelySaved) { + // Abort saving (nothing was "committed" or persisted, the signed urls were not used). + return { wasSaved: false, fileMetadata: fileMetadata }; + } + + const newVersion = await commitProjectVersion({ authenticatedUser, - project, + presignedUrl, + zippedProject, cloudProjectId, options, }); @@ -136,14 +231,22 @@ export const generateOnChangeProjectProperty = ( ): Promise => { if (!authenticatedUser.authenticated) return null; try { - await updateCloudProject( + const [, { presignedUrl, zippedProject }] = await Promise.all([ + updateCloudProject( + authenticatedUser, + fileMetadata.fileIdentifier, + properties + ), + zipAndPrepareProjectVersionForCommit({ + authenticatedUser, + project, + cloudProjectId: fileMetadata.fileIdentifier, + }), + ]); + const newVersion = await commitProjectVersion({ authenticatedUser, - fileMetadata.fileIdentifier, - properties - ); - const newVersion = await zipProjectAndCommitVersion({ - authenticatedUser, - project, + presignedUrl, + zippedProject, cloudProjectId: fileMetadata.fileIdentifier, }); if (!newVersion) { @@ -277,11 +380,20 @@ export const generateOnSaveProjectAs = ( // Commit the changes to the newly created cloud project. await getCredentialsForCloudProject(authenticatedUser, cloudProjectId); - const newVersion = await zipProjectAndCommitVersion({ + const { + presignedUrl, + zippedProject, + } = await zipAndPrepareProjectVersionForCommit({ authenticatedUser, project, cloudProjectId, }); + const newVersion = await commitProjectVersion({ + authenticatedUser, + presignedUrl, + zippedProject, + cloudProjectId, + }); if (!newVersion) throw new Error('No version id was returned from committing api call.'); @@ -355,3 +467,62 @@ export const generateOnAutoSaveProject = ( ); } : undefined; + +const canFileMetadataBeSafelySaved = async ( + authenticatedUser: AuthenticatedUser, + fileMetadata: FileMetadata, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} +) => { + // If the project is saved on the cloud, first fetch it. + // If the version of the project opened is different than the last version of the cloud project, + // it means that the project was modified by someone else. In this case, we should warn + // the user and ask them if they want to overwrite the changes. + const cloudProjectId = fileMetadata.fileIdentifier; + const openedProjectVersion = fileMetadata.version; + const cloudProject: ?CloudProjectWithUserAccessInfo = await getCloudProject( + authenticatedUser, + cloudProjectId + ); + if (!cloudProject) { + await actions.showAlert({ + title: t`Unable to save the project`, + message: t`The project could not be saved. Please try again later.`, + }); + return false; + } + const { currentVersion, committedAt } = cloudProject; + if ( + openedProjectVersion && + currentVersion && // should always be defined. + committedAt && // should always be defined. + currentVersion !== openedProjectVersion + ) { + let lastUsernameWhoModifiedProject = null; + const committedAtDate = new Date(committedAt); + const formattedDate = format(committedAtDate, 'dd-MM-yyyy'); + const formattedTime = format(committedAtDate, 'HH:mm:ss'); + const lastCommittedBy = cloudProject.lastCommittedBy; + if (lastCommittedBy) { + const lastUser = await getUserPublicProfile(lastCommittedBy); + if (lastUser) { + lastUsernameWhoModifiedProject = lastUser.username; + } + } + const answer = await actions.showConfirmation({ + title: t`Project was modified`, + message: lastUsernameWhoModifiedProject + ? t`This project was modified by ${lastUsernameWhoModifiedProject} on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?` + : t`This project was modified by someone else on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?`, + level: 'warning', + confirmButtonLabel: t`Overwrite`, + makeDismissButtonPrimary: true, + }); + + return answer; + } + + return true; +}; diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js index 4e31ec0155da..eb3321546811 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js @@ -1,7 +1,7 @@ // @flow import * as React from 'react'; import { t } from '@lingui/macro'; -import { type StorageProvider, type FileMetadata } from '../index'; +import { type StorageProvider } from '../index'; import { generateOnChangeProjectProperty, generateOnSaveProject, @@ -25,16 +25,6 @@ import { } from './CloudProjectOpener'; import Cloud from '../../UI/CustomSvgIcons/Cloud'; import { generateGetResourceActions } from './CloudProjectResourcesHandler'; -import { - type ShowAlertFunction, - type ShowConfirmFunction, -} from '../../UI/Alert/AlertContext'; -import { - getCloudProject, - type CloudProjectWithUserAccessInfo, -} from '../../Utils/GDevelopServices/Project'; -import { format } from 'date-fns'; -import { getUserPublicProfile } from '../../Utils/GDevelopServices/User'; const isURL = (filename: string) => { return ( @@ -89,63 +79,6 @@ export default ({ return t`An error occurred when opening the project. Check that your internet connection is working and that your browser allows the use of cookies.`; }, getWriteErrorMessage, - canFileMetadataBeSafelySaved: async ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => { - // If the project is saved on the cloud, first fetch it. - // If the version of the project opened is different than the last version of the cloud project, - // it means that the project was modified by someone else. In this case, we should warn - // the user and ask them if they want to overwrite the changes. - const cloudProjectId = fileMetadata.fileIdentifier; - const openedProjectVersion = fileMetadata.version; - const cloudProject: ?CloudProjectWithUserAccessInfo = await getCloudProject( - authenticatedUser, - cloudProjectId - ); - if (!cloudProject) { - await actions.showAlert({ - title: t`Unable to save the project`, - message: t`The project could not be saved. Please try again later.`, - }); - return false; - } - const { currentVersion, committedAt } = cloudProject; - if ( - openedProjectVersion && - currentVersion && // should always be defined. - committedAt && // should always be defined. - currentVersion !== openedProjectVersion - ) { - let lastUsernameWhoModifiedProject = null; - const committedAtDate = new Date(committedAt); - const formattedDate = format(committedAtDate, 'dd-MM-yyyy'); - const formattedTime = format(committedAtDate, 'HH:mm:ss'); - const lastCommittedBy = cloudProject.lastCommittedBy; - if (lastCommittedBy) { - const lastUser = await getUserPublicProfile(lastCommittedBy); - if (lastUser) { - lastUsernameWhoModifiedProject = lastUser.username; - } - } - const answer = await actions.showConfirmation({ - title: t`Project was modified`, - message: lastUsernameWhoModifiedProject - ? t`This project was modified by ${lastUsernameWhoModifiedProject} on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?` - : t`This project was modified by someone else on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?`, - level: 'warning', - confirmButtonLabel: t`Overwrite`, - makeDismissButtonPrimary: true, - }); - - return answer; - } - - return true; - }, }), createResourceOperations: generateGetResourceActions, }: StorageProvider); diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js index bc5542e81fb7..309db68b9e51 100644 --- a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js @@ -1,7 +1,11 @@ // @flow import { t, Trans } from '@lingui/macro'; import * as React from 'react'; -import { serializeToJSObject, serializeToJSON } from '../../Utils/Serializer'; +import { + serializeToJSObject, + serializeToJSON, + serializeToJSObjectInBackground, +} from '../../Utils/Serializer'; import { type FileMetadata, type SaveAsLocation, @@ -16,6 +20,10 @@ import { import type { MessageDescriptor } from '../../Utils/i18n/MessageDescriptor.flow'; import LocalFolderPicker from '../../UI/LocalFolderPicker'; import SaveAsOptionsDialog from '../SaveAsOptionsDialog'; +import { + type ShowAlertFunction, + type ShowConfirmFunction, +} from '../../UI/Alert/AlertContext'; const fs = optionalRequire('fs-extra'); const path = optionalRequire('path'); @@ -102,61 +110,96 @@ const writeAndCheckFormattedJSONFile = async ( await writeAndCheckFile(content, filePath); }; -const writeProjectFiles = ( +const writeProjectFiles = async ( project: gdProject, filePath: string, projectPath: string ): Promise => { - const serializedProjectObject = serializeToJSObject(project); - if (project.isFolderProject()) { - const partialObjects = split(serializedProjectObject, { - pathSeparator: '/', - getArrayItemReferenceName: getSlugifiedUniqueNameFromProperty('name'), - shouldSplit: splitPaths( - new Set( - splittedProjectFolderNames.map(folderName => `/${folderName}/*`) - ) - ), - isReferenceMagicPropertyName: '__REFERENCE_TO_SPLIT_OBJECT', - }); + console.log('--- writeProjectFiles started'); + + const startTime2 = Date.now(); + serializeToJSObject(project); + console.log( + '--- serializeToJSObject done in: ', + Date.now() - startTime2, + 'ms (all on the main thread)' + ); - return Promise.all( - partialObjects.map(partialObject => { - return writeAndCheckFormattedJSONFile( - partialObject.object, - path.join(projectPath, partialObject.reference) + '.json' - ).catch(err => { - console.error('Unable to write a partial file:', err); - throw err; - }); - }) - ).then(() => { - return writeAndCheckFormattedJSONFile( - serializedProjectObject, - filePath - ).catch(err => { - console.error('Unable to write the split project:', err); - throw err; - }); - }); - } else { - return writeAndCheckFormattedJSONFile( - serializedProjectObject, - filePath - ).catch(err => { - console.error('Unable to write the project:', err); - throw err; - }); - } + const startTime = Date.now(); + const serializedProjectObject = await serializeToJSObjectInBackground( + project + ); + console.log( + '--- serializeToJSObjectInBackground done in: ', + Date.now() - startTime, + 'ms (in total, including worker promise)' + ); + + // if (project.isFolderProject()) { + // const partialObjects = split(serializedProjectObject, { + // pathSeparator: '/', + // getArrayItemReferenceName: getSlugifiedUniqueNameFromProperty('name'), + // shouldSplit: splitPaths( + // new Set( + // splittedProjectFolderNames.map(folderName => `/${folderName}/*`) + // ) + // ), + // isReferenceMagicPropertyName: '__REFERENCE_TO_SPLIT_OBJECT', + // }); + + // return Promise.all( + // partialObjects.map(partialObject => { + // return writeAndCheckFormattedJSONFile( + // partialObject.object, + // path.join(projectPath, partialObject.reference) + '.json' + // ).catch(err => { + // console.error('Unable to write a partial file:', err); + // throw err; + // }); + // }) + // ).then(() => { + // return writeAndCheckFormattedJSONFile( + // serializedProjectObject, + // filePath + // ).catch(err => { + // console.error('Unable to write the split project:', err); + // throw err; + // }); + // }); + // } else { + // return writeAndCheckFormattedJSONFile( + // serializedProjectObject, + // filePath + // ).catch(err => { + // console.error('Unable to write the project:', err); + // throw err; + // }); + // } }; export const onSaveProject = async ( project: gdProject, - fileMetadata: FileMetadata + fileMetadata: FileMetadata, + unusedSaveOptions?: {| + previousVersion?: string, + restoredFromVersionId?: string, + |}, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} ): Promise<{| wasSaved: boolean, fileMetadata: FileMetadata, |}> => { + const canBeSafelySaved = await canFileMetadataBeSafelySaved( + fileMetadata, + actions + ); + if (!canBeSafelySaved) { + return { wasSaved: false, fileMetadata: fileMetadata }; + } + const filePath = fileMetadata.fileIdentifier; const now = Date.now(); if (!filePath) { @@ -392,7 +435,7 @@ export const renderNewProjectSaveAsLocationChooser = ({ ); }; -export const isTryingToSaveInForbiddenPath = (filePath: string): boolean => { +const isTryingToSaveInForbiddenPath = (filePath: string): boolean => { if (!remote) return false; // This should not happen, but let's be safe. // If the user is saving locally and chose the same location as where the // executable is running, prevent this, as it will be deleted when the app is updated. @@ -401,3 +444,43 @@ export const isTryingToSaveInForbiddenPath = (filePath: string): boolean => { const gdevelopDirectory = path.dirname(exePath); return filePath.startsWith(gdevelopDirectory); }; + +export const canFileMetadataBeSafelySaved = async ( + fileMetadata: FileMetadata, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} +) => { + const path = fileMetadata.fileIdentifier; + if (isTryingToSaveInForbiddenPath(path)) { + await actions.showAlert({ + title: t`Choose another location`, + message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, + }); + } + + // We don't block the save, in case the user wants to save anyway. + return true; +}; + +export const canFileMetadataBeSafelySavedAs = async ( + fileMetadata: FileMetadata, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} +) => { + const path = fileMetadata.fileIdentifier; + if (isTryingToSaveInForbiddenPath(path)) { + await actions.showAlert({ + title: t`Choose another location`, + message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, + }); + + // We block the save as we don't want new projects to be saved there. + return false; + } + + return true; +}; diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js index d691614a96fa..a82f98bed3e6 100644 --- a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js @@ -1,7 +1,7 @@ // @flow import * as React from 'react'; import { Trans, t } from '@lingui/macro'; -import { type StorageProvider, type FileMetadata } from '../index'; +import { type StorageProvider } from '../index'; import { onOpenWithPicker, onOpen, @@ -16,7 +16,7 @@ import { getWriteErrorMessage, renderNewProjectSaveAsLocationChooser, getProjectLocation, - isTryingToSaveInForbiddenPath, + canFileMetadataBeSafelySavedAs, } from './LocalProjectWriter'; import { type AppArguments, @@ -32,10 +32,6 @@ import { scanForNewResources, } from './LocalProjectResourcesHandler'; import { allResourceKindsAndMetadata } from '../../ResourcesList/ResourceSource'; -import { - type ShowAlertFunction, - type ShowConfirmFunction, -} from '../../UI/Alert/AlertContext'; import { setupResourcesWatcher } from './LocalFileResourcesWatcher'; /** @@ -73,44 +69,7 @@ export default ({ return t`Check that the file exists, that this file is a proper game created with GDevelop and that you have the authorization to open it.`; }, getWriteErrorMessage, - canFileMetadataBeSafelySaved: async ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => { - const path = fileMetadata.fileIdentifier; - if (isTryingToSaveInForbiddenPath(path)) { - await actions.showAlert({ - title: t`Choose another location`, - message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, - }); - } - - // We don't block the save, in case the user wants to save anyway. - return true; - }, - canFileMetadataBeSafelySavedAs: async ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => { - const path = fileMetadata.fileIdentifier; - if (isTryingToSaveInForbiddenPath(path)) { - await actions.showAlert({ - title: t`Choose another location`, - message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, - }); - - // We block the save as we don't want new projects to be saved there. - return false; - } - - return true; - }, + canFileMetadataBeSafelySavedAs, }), createResourceOperations: () => ({ project, diff --git a/newIDE/app/src/ProjectsStorage/index.js b/newIDE/app/src/ProjectsStorage/index.js index 60e29269283b..e2590f913ece 100644 --- a/newIDE/app/src/ProjectsStorage/index.js +++ b/newIDE/app/src/ProjectsStorage/index.js @@ -106,7 +106,11 @@ export type StorageProviderOperations = {| onSaveProject?: ( project: gdProject, fileMetadata: FileMetadata, - options?: {| previousVersion?: string, restoredFromVersionId?: string |} + options?: {| previousVersion?: string, restoredFromVersionId?: string |}, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} ) => Promise<{| wasSaved: boolean, fileMetadata: FileMetadata, @@ -133,13 +137,6 @@ export type StorageProviderOperations = {| /** This is the location where the project was saved, or null if not persisted. */ fileMetadata: ?FileMetadata, |}>, - canFileMetadataBeSafelySaved?: ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => Promise, canFileMetadataBeSafelySavedAs?: ( fileMetadata: FileMetadata, actions: {| diff --git a/newIDE/app/src/Utils/GDevelopServices/Project.js b/newIDE/app/src/Utils/GDevelopServices/Project.js index 363113d731f3..bd55656557c4 100644 --- a/newIDE/app/src/Utils/GDevelopServices/Project.js +++ b/newIDE/app/src/Utils/GDevelopServices/Project.js @@ -298,12 +298,14 @@ export const createCloudProject = async ( export const commitVersion = async ({ authenticatedUser, cloudProjectId, + presignedUrl, zippedProject, previousVersion, restoredFromVersionId, }: { authenticatedUser: AuthenticatedUser, cloudProjectId: string, + presignedUrl: string, zippedProject: Blob, previousVersion?: ?string, restoredFromVersionId?: ?string, @@ -313,12 +315,7 @@ export const commitVersion = async ({ const { uid: userId } = firebaseUser; const authorizationHeader = await getAuthorizationHeader(); - // Get a presigned url to upload a new version (the URL will contain the new version id). - const presignedUrl = await getPresignedUrlForVersionUpload( - authenticatedUser, - cloudProjectId - ); - if (!presignedUrl) return; + const newVersion = getVersionIdFromPath(presignedUrl); // Upload zipped project. await refetchCredentialsForProjectAndRetryIfUnauthorized( @@ -552,7 +549,7 @@ export const deleteCloudProject = async ( }); }; -const getPresignedUrlForVersionUpload = async ( +export const getPresignedUrlForVersionUpload = async ( authenticatedUser: AuthenticatedUser, cloudProjectId: string ): Promise => { diff --git a/newIDE/app/src/Utils/Serializer.js b/newIDE/app/src/Utils/Serializer.js index b026c224ee0e..997dcef403da 100644 --- a/newIDE/app/src/Utils/Serializer.js +++ b/newIDE/app/src/Utils/Serializer.js @@ -1,5 +1,9 @@ // @flow +import VersionMetadata from '../Version/VersionMetadata'; +// Import the worker (handled by worker-loader) +import SerializerWorker from './Serializer.worker'; + const gd: libGDevelop = global.gd; /** @@ -14,15 +18,37 @@ export function serializeToJSObject( serializable: gdSerializable, methodName: string = 'serializeTo' ) { + const startTime = Date.now(); const serializedElement = new gd.SerializerElement(); serializable[methodName](serializedElement); + const serializeToTime = Date.now(); + console.log( + '[Main thread] serializeTo done in: ', + serializeToTime - startTime, + 'ms' + ); // JSON.parse + toJSON is 30% faster than gd.Serializer.toJSObject. + const toJSONStartTime = Date.now(); const json = gd.Serializer.toJSON(serializedElement); + const toJSONTime = Date.now(); + console.log( + '[Main thread] toJSON done in: ', + toJSONTime - toJSONStartTime, + 'ms' + ); + try { const object = JSON.parse(json); + const parseTime = Date.now(); + console.log('[Main thread] parse done in: ', parseTime - toJSONTime, 'ms'); serializedElement.delete(); + console.log( + '[Main thread] serializeToJSObject done in: ', + Date.now() - startTime, + 'ms' + ); return object; } catch (error) { serializedElement.delete(); @@ -98,16 +124,166 @@ export function serializeToJSON( serializable: gdSerializable, methodName: string = 'serializeTo' ): string { + const startTime = Date.now(); const serializedElement = new gd.SerializerElement(); serializable[methodName](serializedElement); + const serializeToTime = Date.now(); + console.log( + '[Main thread] serializeTo done in: ', + serializeToTime - startTime, + 'ms' + ); // toJSON is 20% faster than gd.Serializer.toJSObject + JSON.stringify. const json = gd.Serializer.toJSON(serializedElement); serializedElement.delete(); + const toJSONTime = Date.now(); + console.log( + '[Main thread] toJSON done in: ', + toJSONTime - serializeToTime, + 'ms' + ); return json; } +type SerializerWorkerOutMessage = + | {| type: 'serialized', object: any |} + | {| type: 'error', message: string |}; + +let serializerWorker: ?Worker = null; + +const getOrCreateSerializerWorker = (): Worker => { + if (!serializerWorker) { + // $FlowExpectedError - worker-loader types aren't recognized by Flow + serializerWorker = new SerializerWorker(); + } + + return serializerWorker; +}; + +export function terminateSerializerWorkerForTests() { + if (serializerWorker) { + serializerWorker.terminate(); + serializerWorker = null; + } +} + +/** + * Serialize a gdSerializable into a JS object while keeping the slowest parts + * off the main thread. + */ +export async function serializeToJSObjectInBackground( + serializable: gdSerializable, + methodName: string = 'serializeTo' +): Promise { + const startTime = Date.now(); + const serializedElement = new gd.SerializerElement(); + serializable[methodName](serializedElement); + const serializeToTime = Date.now(); + console.log( + '[Main thread] serializeTo done in: ', + serializeToTime - startTime, + 'ms' + ); + + // Allocate space for the output size in WASM heap. + // $FlowExpectedError - accessing emscripten memory helpers. + const sizePtr = gd._malloc(4); + if (!sizePtr) { + serializedElement.delete(); + throw new Error('Failed to allocate memory for serialization size.'); + } + + let binaryPtr = 0; + try { + // $FlowExpectedError - accessing emscripten exports. + binaryPtr = gd._createBinarySnapshot(serializedElement.ptr, sizePtr); + if (!binaryPtr) { + throw new Error('Failed to create binary snapshot.'); + } + + const createBinarySnapshotTime = Date.now(); + console.log( + '[Main thread] createBinarySnapshot done in: ', + createBinarySnapshotTime - serializeToTime, + 'ms' + ); + + // $FlowExpectedError - accessing emscripten memory helpers. + const binarySize = gd.HEAPU32[sizePtr >> 2]; + // $FlowExpectedError - accessing emscripten memory helpers. + const binaryView = new Uint8Array(gd.HEAPU8.buffer, binaryPtr, binarySize); + // Copy the buffer out of the WASM heap so it can be transferred. + const binaryBuffer = binaryView.slice(); + + const binaryBufferTime = Date.now(); + console.log( + '[Main thread] BinaryBuffer copied/prepared in: ', + binaryBufferTime - createBinarySnapshotTime, + 'ms' + ); + + const worker = getOrCreateSerializerWorker(); + const cacheBuster = VersionMetadata.versionWithHash; + + const workerPromiseStartTime = Date.now(); + const object = await new Promise((resolve, reject) => { + const handleMessage = (event: MessageEvent) => { + const data: SerializerWorkerOutMessage = event.data; + if (data.type === 'serialized') { + worker.removeEventListener('message', handleMessage); + worker.removeEventListener('error', handleError); + resolve(data.object); + } else if (data.type === 'error') { + worker.removeEventListener('message', handleMessage); + worker.removeEventListener('error', handleError); + reject(new Error(data.message)); + } + }; + + const handleError = (error: any) => { + worker.removeEventListener('message', handleMessage); + worker.removeEventListener('error', handleError); + reject(error); + }; + + worker.addEventListener('message', handleMessage); + worker.addEventListener('error', handleError); + worker.postMessage( + { + type: 'serialize', + binary: binaryBuffer, + cacheBuster, + }, + [binaryBuffer.buffer] + ); + console.log( + '[Main thread] main thread work done in: ', + Date.now() - startTime, + 'ms' + ); + }); + + const workerPromiseTime = Date.now(); + console.log( + '[Main thread] The worker promise (doing JSON serialization from the binary buffer) returned in: ', + workerPromiseTime - workerPromiseStartTime, + 'ms' + ); + + return object; + } finally { + if (binaryPtr) { + // $FlowExpectedError - accessing emscripten exports. + gd._freeBinarySnapshot(binaryPtr); + } + // $FlowExpectedError - accessing emscripten exports. + gd._free(sizePtr); + serializedElement.delete(); + } +} + /** * Tool function to restore a serializable object from a JS object. * Most gd.* objects are "serializable", meaning they have a serializeTo diff --git a/newIDE/app/src/Utils/Serializer.worker.js b/newIDE/app/src/Utils/Serializer.worker.js new file mode 100644 index 000000000000..0db922d17a54 --- /dev/null +++ b/newIDE/app/src/Utils/Serializer.worker.js @@ -0,0 +1,104 @@ +/* eslint-env worker */ +// @flow + +let modulePromise /*: ?Promise*/ = null; + +const ensureLibGD = (cacheBuster /*: ?string*/) => { + if (modulePromise) return modulePromise; + + modulePromise = new Promise((resolve, reject) => { + try { + const libGDUrl = cacheBuster + ? `/libGD.js?cache-buster=${cacheBuster}` + : '/libGD.js'; + // Load libGD.js in the worker context. + // eslint-disable-next-line no-undef + importScripts(libGDUrl); + + // eslint-disable-next-line no-undef + if (typeof initializeGDevelopJs !== 'function') { + reject(new Error('Missing initializeGDevelopJs in worker')); + return; + } + + // eslint-disable-next-line no-undef + initializeGDevelopJs({ + locateFile: (path /*: string*/) => + cacheBuster ? `/${path}?cache-buster=${cacheBuster}` : `/${path}`, + }) + .then(module => { + resolve(module); + }) + .catch(reject); + } catch (error) { + reject(error); + return; + } + }); + + return modulePromise; +}; + +// eslint-disable-next-line no-restricted-globals +self.onmessage = async (event /*: MessageEvent*/) => { + const { type, binary, cacheBuster } = event.data || {}; + + if (type !== 'serialize') return; + + // TODO: handle request ids + + try { + console.log('Serializer worker: serialize started'); + const gd = await ensureLibGD(cacheBuster); + console.log('Serializer worker: libGD initialized'); + + const binaryArray = + binary instanceof Uint8Array ? binary : new Uint8Array(binary); + const binarySize = binaryArray.byteLength || binaryArray.length; + + const startTime = Date.now(); + + const binaryPtr = gd._malloc(binarySize); + gd.HEAPU8.set(binaryArray, binaryPtr); + + const elementPtr = gd._deserializeBinarySnapshot(binaryPtr, binarySize); + if (!elementPtr) { + gd._free(binaryPtr); + // eslint-disable-next-line no-restricted-globals + self.postMessage({ + type: 'error', + message: 'Failed to deserialize binary snapshot', + }); + return; + } + + const element = + typeof gd.wrapPointer === 'function' + ? gd.wrapPointer(elementPtr, gd.SerializerElement) + : new gd.SerializerElement(elementPtr); + + const json = gd.Serializer.toJSON(element); + const object = JSON.parse(json); + + gd._free(binaryPtr); + element.delete(); + + console.log( + 'Serializer worker: serialize finished in', + Date.now() - startTime, + 'ms' + ); + // eslint-disable-next-line no-restricted-globals + self.postMessage({ + type: 'serialized', + object, + duration: Date.now() - startTime, + }); + } catch (error) { + // eslint-disable-next-line no-restricted-globals + self.postMessage({ + type: 'error', + message: error.message, + }); + } +}; From 09728ec956dc6afe3c8fe6f8c66b9e80f42bdb55 Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Mon, 22 Dec 2025 16:59:57 +0100 Subject: [PATCH 02/10] Refactor --- .../GDCore/Serialization/BinarySerializer.cpp | 53 ++++--- Core/GDCore/Serialization/BinarySerializer.h | 71 ++++----- GDevelop.js/Bindings/Bindings.idl | 15 ++ GDevelop.js/Bindings/Wrapper.cpp | 5 + GDevelop.js/CMakeLists.txt | 2 +- GDevelop.js/scripts/generate-dts.mjs | 4 + GDevelop.js/scripts/generate-types.js | 5 + GDevelop.js/types.d.ts | 11 ++ GDevelop.js/types/gdbinaryserializer.js | 9 ++ GDevelop.js/types/libgdevelop.js | 6 + newIDE/app/scripts/make-service-worker.js | 2 +- .../CloudProjectWriter.js | 23 +-- .../LocalProjectWriter.js | 123 ++++++++------- newIDE/app/src/Utils/BackgroundSerializer.js | 131 ++++++++++++++++ .../src/Utils/BackgroundSerializer.worker.js | 115 ++++++++++++++ newIDE/app/src/Utils/Serializer.js | 142 ------------------ newIDE/app/src/Utils/Serializer.worker.js | 104 ------------- 17 files changed, 452 insertions(+), 369 deletions(-) create mode 100644 GDevelop.js/types/gdbinaryserializer.js create mode 100644 newIDE/app/src/Utils/BackgroundSerializer.js create mode 100644 newIDE/app/src/Utils/BackgroundSerializer.worker.js delete mode 100644 newIDE/app/src/Utils/Serializer.worker.js diff --git a/Core/GDCore/Serialization/BinarySerializer.cpp b/Core/GDCore/Serialization/BinarySerializer.cpp index 0db21d856523..81e557fae03c 100644 --- a/Core/GDCore/Serialization/BinarySerializer.cpp +++ b/Core/GDCore/Serialization/BinarySerializer.cpp @@ -11,9 +11,11 @@ namespace gd { +size_t BinarySerializer::lastBinarySnapshotSize = 0; + using NodeType = BinarySerializer::NodeType; -void BinarySerializer::SerializeToBinary(const SerializerElement& element, +void BinarySerializer::SerializeToBinaryBuffer(const SerializerElement& element, std::vector& outBuffer) { // Reserve approximate size (heuristic: 1KB minimum) outBuffer.clear(); @@ -89,7 +91,7 @@ void BinarySerializer::SerializeString(const gd::String& str, buffer.insert(buffer.end(), utf8.begin(), utf8.end()); } -bool BinarySerializer::DeserializeFromBinary(const uint8_t* buffer, +bool BinarySerializer::DeserializeFromBinaryBuffer(const uint8_t* buffer, size_t bufferSize, SerializerElement& outElement) { const uint8_t* ptr = buffer; @@ -234,31 +236,43 @@ bool BinarySerializer::DeserializeString(const uint8_t*& ptr, return true; } -#ifdef EMSCRIPTEN -extern "C" { +uintptr_t BinarySerializer::CreateBinarySnapshot(const SerializerElement& element) { + std::vector buffer; + SerializeToBinaryBuffer(element, buffer); -EMSCRIPTEN_KEEPALIVE uint8_t* createBinarySnapshot(SerializerElement* element, - size_t* outSize) { - if (!element || !outSize) return nullptr; + lastBinarySnapshotSize = buffer.size(); - std::vector buffer; - gd::BinarySerializer::SerializeToBinary(*element, buffer); + // Allocate memory in Emscripten heap + uint8_t* heapBuffer = (uint8_t*)malloc(buffer.size()); + if (!heapBuffer) { + lastBinarySnapshotSize = 0; + return 0; + } - *outSize = buffer.size(); - uint8_t* result = (uint8_t*)malloc(buffer.size()); - std::memcpy(result, buffer.data(), buffer.size()); + std::memcpy(heapBuffer, buffer.data(), buffer.size()); + return reinterpret_cast(heapBuffer); +} - return result; +size_t BinarySerializer::GetLastBinarySnapshotSize() { + return lastBinarySnapshotSize; } -EMSCRIPTEN_KEEPALIVE void freeBinarySnapshot(uint8_t* buffer) { free(buffer); } +void BinarySerializer::FreeBinarySnapshot(uintptr_t bufferPtr) { + if (bufferPtr) { + free(reinterpret_cast(bufferPtr)); + } +} -EMSCRIPTEN_KEEPALIVE SerializerElement* deserializeBinarySnapshot( - const uint8_t* buffer, size_t size) { - if (!buffer || size == 0) return nullptr; +SerializerElement* BinarySerializer::DeserializeBinarySnapshot(uintptr_t bufferPtr, + size_t size) { + if (!bufferPtr || size == 0) { + return nullptr; + } + const uint8_t* buffer = reinterpret_cast(bufferPtr); SerializerElement* element = new SerializerElement(); - if (!gd::BinarySerializer::DeserializeFromBinary(buffer, size, *element)) { + + if (!DeserializeFromBinaryBuffer(buffer, size, *element)) { delete element; return nullptr; } @@ -266,7 +280,4 @@ EMSCRIPTEN_KEEPALIVE SerializerElement* deserializeBinarySnapshot( return element; } -} // extern "C" -#endif - } // namespace gd diff --git a/Core/GDCore/Serialization/BinarySerializer.h b/Core/GDCore/Serialization/BinarySerializer.h index 154ade88c97b..28295be0e1a5 100644 --- a/Core/GDCore/Serialization/BinarySerializer.h +++ b/Core/GDCore/Serialization/BinarySerializer.h @@ -1,11 +1,10 @@ /* * GDevelop Core - * Copyright 2008-2016 Florian Rival (Florian.Rival@gmail.com). All rights + * Copyright 2008-present Florian Rival (Florian.Rival@gmail.com). All rights * reserved. This project is released under the MIT License. */ -#ifndef GDCORE_BINARY_SERIALIZER_H -#define GDCORE_BINARY_SERIALIZER_H +#pragma once #include #include @@ -19,7 +18,7 @@ namespace gd { * \brief Fast binary serialization/deserialization for SerializerElement trees. * * This format is optimized for speed and compactness, not for human readability - * or long-term storage. Use it for transferring data between threads. + * or long-term storage. For transferring data between "threads" (web workers). */ class GD_CORE_API BinarySerializer { public: @@ -29,7 +28,7 @@ class GD_CORE_API BinarySerializer { * \param element The root element to serialize * \param outBuffer Output buffer that will contain the binary data */ - static void SerializeToBinary(const SerializerElement& element, + static void SerializeToBinaryBuffer(const SerializerElement& element, std::vector& outBuffer); /** @@ -40,10 +39,39 @@ class GD_CORE_API BinarySerializer { * \param outElement The output element (will be populated) * \return true if successful, false if corrupted data */ - static bool DeserializeFromBinary(const uint8_t* buffer, + static bool DeserializeFromBinaryBuffer(const uint8_t* buffer, size_t bufferSize, SerializerElement& outElement); + /** \name Helpers + */ + ///@{ + /** + * \brief Create a binary snapshot and return the size. + * The binary data is written to Emscripten's heap at the returned pointer. + * \return Pointer to binary data in Emscripten heap (use GetLastBinarySnapshotSize to get size) + */ + static uintptr_t CreateBinarySnapshot(const SerializerElement& element); + + /** + * \brief Get the size of the last created binary snapshot. + * Must be called immediately after CreateBinarySnapshot. + */ + static size_t GetLastBinarySnapshotSize(); + + /** + * \brief Free a binary snapshot. + */ + static void FreeBinarySnapshot(uintptr_t bufferPtr); + + /** + * \brief Deserialize binary snapshot back to SerializerElement. + * \return New SerializerElement pointer (caller owns it) + */ + static SerializerElement* DeserializeBinarySnapshot(uintptr_t bufferPtr, + size_t size); + ///@} + enum class NodeType : uint8_t { Element = 0x01, ValueUndefined = 0x02, @@ -89,34 +117,9 @@ class GD_CORE_API BinarySerializer { ptr += sizeof(T); return true; } -}; - -// Emscripten bindings -#ifdef EMSCRIPTEN -#include - -extern "C" { -/** - * \brief Create a binary snapshot of a SerializerElement. - * \return Pointer to binary data (caller must free with freeBinarySnapshot) - */ -EMSCRIPTEN_KEEPALIVE uint8_t* createBinarySnapshot(SerializerElement* element, - size_t* outSize); - -/** - * \brief Free a binary snapshot created by createBinarySnapshot. - */ -EMSCRIPTEN_KEEPALIVE void freeBinarySnapshot(uint8_t* buffer); -/** - * \brief Deserialize binary snapshot back to SerializerElement. - * \return New SerializerElement (caller must delete) - */ -EMSCRIPTEN_KEEPALIVE SerializerElement* deserializeBinarySnapshot( - const uint8_t* buffer, size_t size); -} -#endif + // Store the last binary size for retrieval + static size_t lastBinarySnapshotSize; +}; } // namespace gd - -#endif // GDCORE_BINARY_SERIALIZER_H diff --git a/GDevelop.js/Bindings/Bindings.idl b/GDevelop.js/Bindings/Bindings.idl index 5ddbe2d01686..108dc555a4c9 100644 --- a/GDevelop.js/Bindings/Bindings.idl +++ b/GDevelop.js/Bindings/Bindings.idl @@ -1603,6 +1603,21 @@ interface Serializer { [Value] SerializerElement STATIC_FromJSON([Const] DOMString json); }; +[NoDelete] +interface BinarySerializer { + // Create a binary snapshot, returns pointer to buffer in Emscripten heap + static unsigned long STATIC_CreateBinarySnapshot([Ref] SerializerElement element); + + // Get the size of the last created snapshot + static unsigned long STATIC_GetLastBinarySnapshotSize(); + + // Free the binary snapshot + static void STATIC_FreeBinarySnapshot(unsigned long bufferPtr); + + // Deserialize from a pointer in Emscripten heap + static SerializerElement STATIC_DeserializeBinarySnapshot(unsigned long bufferPtr, unsigned long size); +}; + interface ObjectAssetSerializer { void STATIC_SerializeTo([Ref] Project project, [Const, Ref] gdObject obj, [Const] DOMString objectFullName, [Ref] SerializerElement element, diff --git a/GDevelop.js/Bindings/Wrapper.cpp b/GDevelop.js/Bindings/Wrapper.cpp index f5f45d1324d0..63176d39e2b7 100644 --- a/GDevelop.js/Bindings/Wrapper.cpp +++ b/GDevelop.js/Bindings/Wrapper.cpp @@ -99,6 +99,7 @@ #include #include #include +#include #include #include #include @@ -855,6 +856,10 @@ typedef std::vector VectorPropertyDescriptorChoice #define STATIC_GetDefaultMeasurementUnitByName GetDefaultMeasurementUnitByName #define STATIC_HasDefaultMeasurementUnitNamed HasDefaultMeasurementUnitNamed #define STATIC_GetEdgeAnchorFromString GetEdgeAnchorFromString +#define STATIC_CreateBinarySnapshot CreateBinarySnapshot +#define STATIC_GetLastBinarySnapshotSize GetLastBinarySnapshotSize +#define STATIC_FreeBinarySnapshot FreeBinarySnapshot +#define STATIC_DeserializeBinarySnapshot DeserializeBinarySnapshot // We postfix some methods with "At" as Javascript does not support overloading #define GetLayoutAt GetLayout diff --git a/GDevelop.js/CMakeLists.txt b/GDevelop.js/CMakeLists.txt index 47fb0df089f9..fb0841083021 100644 --- a/GDevelop.js/CMakeLists.txt +++ b/GDevelop.js/CMakeLists.txt @@ -91,7 +91,7 @@ target_link_libraries(GD "-s TOTAL_MEMORY=48MB") # Get some initial memory size target_link_libraries(GD "-s ALLOW_MEMORY_GROWTH=1") target_link_libraries(GD "-s NODEJS_CATCH_EXIT=0") # Don't print the entire GDCore code on error when running in node target_link_libraries(GD "-s ERROR_ON_UNDEFINED_SYMBOLS=0") -target_link_libraries(GD "-s \"EXPORTED_FUNCTIONS=['_free','_malloc','_createBinarySnapshot','_freeBinarySnapshot','_deserializeBinarySnapshot']\"") +target_link_libraries(GD "-s \"EXPORTED_FUNCTIONS=['_free','_malloc']\"") if("${GDEVELOPJS_BUILD_VARIANT}" STREQUAL "dev") # Disable optimizations at linking time for slightly faster builds. diff --git a/GDevelop.js/scripts/generate-dts.mjs b/GDevelop.js/scripts/generate-dts.mjs index ff036247d226..b04d9fbe4efd 100644 --- a/GDevelop.js/scripts/generate-dts.mjs +++ b/GDevelop.js/scripts/generate-dts.mjs @@ -453,6 +453,10 @@ export function compare(object1: T, object2: T): boo */ export function destroy(object: EmscriptenObject): void; +export function _malloc(size: number): number; +export function _free(ptr: number): void; +export const HEAPU8: Uint8Array; + export as namespace gd; declare global { diff --git a/GDevelop.js/scripts/generate-types.js b/GDevelop.js/scripts/generate-types.js index 3705f1e9357b..715b5fbb5549 100644 --- a/GDevelop.js/scripts/generate-types.js +++ b/GDevelop.js/scripts/generate-types.js @@ -288,8 +288,13 @@ type CustomObjectConfiguration_EdgeAnchor = 0 | 1 | 2 | 3 | 4` 'declare class libGDevelop {', ' getPointer(gdEmscriptenObject): number;', ' castObject(gdEmscriptenObject, Class): T;', + ' wrapPointer(ptr: number, objectClass: Class): T;', ' compare(gdEmscriptenObject, gdEmscriptenObject): boolean;', '', + ' _malloc(size: number): number;', + ' _free(ptr: number): void;', + ' HEAPU8: Uint8Array;', + '', ' getTypeOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string;', ' getTypeOfBehavior(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string;', ' getBehaviorsOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): gdVectorString;', diff --git a/GDevelop.js/types.d.ts b/GDevelop.js/types.d.ts index 4f40c764e896..93a5a764ae7f 100644 --- a/GDevelop.js/types.d.ts +++ b/GDevelop.js/types.d.ts @@ -1325,6 +1325,13 @@ export class Serializer extends EmscriptenObject { static toJSObject(element: gdSerializerElement): any; } +export class BinarySerializer extends EmscriptenObject { + static unsigned long CreateBinarySnapshot(element: SerializerElement): static; + static unsigned long GetLastBinarySnapshotSize(): static; + static void FreeBinarySnapshot(bufferPtr: number): static; + static serializerElement DeserializeBinarySnapshot(bufferPtr: number, size: number): static; +} + export class ObjectAssetSerializer extends EmscriptenObject { static serializeTo(project: Project, obj: gdObject, objectFullName: string, element: SerializerElement, usedResourceNames: VectorString): void; } @@ -3228,6 +3235,10 @@ export function compare(object1: T, object2: T): boo */ export function destroy(object: EmscriptenObject): void; +export function _malloc(size: number): number; +export function _free(ptr: number): void; +export const HEAPU8: Uint8Array; + export as namespace gd; declare global { diff --git a/GDevelop.js/types/gdbinaryserializer.js b/GDevelop.js/types/gdbinaryserializer.js new file mode 100644 index 000000000000..17932d328b73 --- /dev/null +++ b/GDevelop.js/types/gdbinaryserializer.js @@ -0,0 +1,9 @@ +// Automatically generated by GDevelop.js/scripts/generate-types.js +declare class gdBinarySerializer { + static createBinarySnapshot(element: gdSerializerElement): number; + static getLastBinarySnapshotSize(): number; + static freeBinarySnapshot(bufferPtr: number): void; + static deserializeBinarySnapshot(bufferPtr: number, size: number): gdSerializerElement; + delete(): void; + ptr: number; +}; \ No newline at end of file diff --git a/GDevelop.js/types/libgdevelop.js b/GDevelop.js/types/libgdevelop.js index ffc1b62b68fe..eab9afa3582f 100644 --- a/GDevelop.js/types/libgdevelop.js +++ b/GDevelop.js/types/libgdevelop.js @@ -2,8 +2,13 @@ declare class libGDevelop { getPointer(gdEmscriptenObject): number; castObject(gdEmscriptenObject, Class): T; + wrapPointer(ptr: number, objectClass: Class): T; compare(gdEmscriptenObject, gdEmscriptenObject): boolean; + _malloc(size: number): number; + _free(ptr: number): void; + HEAPU8: Uint8Array; + getTypeOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string; getTypeOfBehavior(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string; getBehaviorsOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): gdVectorString; @@ -141,6 +146,7 @@ declare class libGDevelop { SerializerElement: Class; SharedPtrSerializerElement: Class; Serializer: Class; + BinarySerializer: Class; ObjectAssetSerializer: Class; InstructionsList: Class; Instruction: Class; diff --git a/newIDE/app/scripts/make-service-worker.js b/newIDE/app/scripts/make-service-worker.js index 8b4d19bde320..11021c16d729 100644 --- a/newIDE/app/scripts/make-service-worker.js +++ b/newIDE/app/scripts/make-service-worker.js @@ -56,7 +56,7 @@ const buildSW = () => { 'static/js/locales-*.js', // Locales. 'static/js/!local-app*.js', // Exclude electron app. 'static/js/Resource3DPreview.worker.*.js', // Include the 3D preview worker - 'static/js/Serializer.worker.*.js', // Include the serializer worker + 'static/js/BackgroundSerializer.worker.*.js', // Include the serializer worker // ...But not libGD.js/wasm (there are cached with their URL // query string that depends on the VersionMetadata, see below). diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js index 30657e6e2b99..eb0ede3e3b79 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js @@ -13,9 +13,9 @@ import { import type { $AxiosError } from 'axios'; import type { MessageDescriptor } from '../../Utils/i18n/MessageDescriptor.flow'; import { - serializeToJSObjectInBackground, serializeToJSON, } from '../../Utils/Serializer'; +import { serializeToJSONInBackground } from '../../Utils/BackgroundSerializer'; import { t } from '@lingui/macro'; import { createZipWithSingleTextFile, @@ -48,16 +48,17 @@ const zipProject = async (project: gdProject): Promise<[Blob, string]> => { 'ms' ); - // const startTime2 = Date.now(); - // // TODO: should serialize to JSON instead of JS object. - // const serializedProjectObject = await serializeToJSObjectInBackground( - // project - // ); - // console.log( - // '--- serializeToJSObjectInBackground done in: ', - // Date.now() - startTime2, - // 'ms (in total, including worker promise)' - // ); + const startTime2 = Date.now(); + // TODO: should serialize to JSON instead of JS object. + const projectJson2 = await serializeToJSONInBackground(project); + console.log( + '--- serializeToJSONInBackground done in: ', + Date.now() - startTime2, + 'ms (in total, including worker promise)' + ); + if (projectJson2 !== projectJson) { + console.log('Project JSONs are different.', projectJson, projectJson2); + } const zipStartTime = Date.now(); const zippedProject = await createZipWithSingleTextFile( diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js index 309db68b9e51..532d056093d0 100644 --- a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js @@ -1,11 +1,8 @@ // @flow import { t, Trans } from '@lingui/macro'; import * as React from 'react'; -import { - serializeToJSObject, - serializeToJSON, - serializeToJSObjectInBackground, -} from '../../Utils/Serializer'; +import { serializeToJSObject, serializeToJSON } from '../../Utils/Serializer'; +import { serializeToJSObjectInBackground } from '../../Utils/BackgroundSerializer'; import { type FileMetadata, type SaveAsLocation, @@ -118,63 +115,79 @@ const writeProjectFiles = async ( console.log('--- writeProjectFiles started'); const startTime2 = Date.now(); - serializeToJSObject(project); + const serializedProjectObject = serializeToJSObject(project); console.log( '--- serializeToJSObject done in: ', Date.now() - startTime2, 'ms (all on the main thread)' ); - const startTime = Date.now(); - const serializedProjectObject = await serializeToJSObjectInBackground( - project - ); - console.log( - '--- serializeToJSObjectInBackground done in: ', - Date.now() - startTime, - 'ms (in total, including worker promise)' - ); + try { + const startTime = Date.now(); + const serializedProjectObject2 = await serializeToJSObjectInBackground( + project + ); + console.log( + '--- serializeToJSObjectInBackground done in: ', + Date.now() - startTime, + 'ms (in total, including worker promise)' + ); + + if ( + JSON.stringify(serializedProjectObject) !== + JSON.stringify(serializedProjectObject2) + ) { + console.log( + 'Project JSONs are different.', + serializedProjectObject, + serializedProjectObject2 + ); + } + } catch (error) { + console.error('Unable to serialize to JS object in background:', error); + throw error; + } + + if (project.isFolderProject()) { + const partialObjects = split(serializedProjectObject, { + pathSeparator: '/', + getArrayItemReferenceName: getSlugifiedUniqueNameFromProperty('name'), + shouldSplit: splitPaths( + new Set( + splittedProjectFolderNames.map(folderName => `/${folderName}/*`) + ) + ), + isReferenceMagicPropertyName: '__REFERENCE_TO_SPLIT_OBJECT', + }); - // if (project.isFolderProject()) { - // const partialObjects = split(serializedProjectObject, { - // pathSeparator: '/', - // getArrayItemReferenceName: getSlugifiedUniqueNameFromProperty('name'), - // shouldSplit: splitPaths( - // new Set( - // splittedProjectFolderNames.map(folderName => `/${folderName}/*`) - // ) - // ), - // isReferenceMagicPropertyName: '__REFERENCE_TO_SPLIT_OBJECT', - // }); - - // return Promise.all( - // partialObjects.map(partialObject => { - // return writeAndCheckFormattedJSONFile( - // partialObject.object, - // path.join(projectPath, partialObject.reference) + '.json' - // ).catch(err => { - // console.error('Unable to write a partial file:', err); - // throw err; - // }); - // }) - // ).then(() => { - // return writeAndCheckFormattedJSONFile( - // serializedProjectObject, - // filePath - // ).catch(err => { - // console.error('Unable to write the split project:', err); - // throw err; - // }); - // }); - // } else { - // return writeAndCheckFormattedJSONFile( - // serializedProjectObject, - // filePath - // ).catch(err => { - // console.error('Unable to write the project:', err); - // throw err; - // }); - // } + return Promise.all( + partialObjects.map(partialObject => { + return writeAndCheckFormattedJSONFile( + partialObject.object, + path.join(projectPath, partialObject.reference) + '.json' + ).catch(err => { + console.error('Unable to write a partial file:', err); + throw err; + }); + }) + ).then(() => { + return writeAndCheckFormattedJSONFile( + serializedProjectObject, + filePath + ).catch(err => { + console.error('Unable to write the split project:', err); + throw err; + }); + }); + } else { + return writeAndCheckFormattedJSONFile( + serializedProjectObject, + filePath + ).catch(err => { + console.error('Unable to write the project:', err); + throw err; + }); + } }; export const onSaveProject = async ( diff --git a/newIDE/app/src/Utils/BackgroundSerializer.js b/newIDE/app/src/Utils/BackgroundSerializer.js new file mode 100644 index 000000000000..5d72dcded021 --- /dev/null +++ b/newIDE/app/src/Utils/BackgroundSerializer.js @@ -0,0 +1,131 @@ +// @flow + +import VersionMetadata from '../Version/VersionMetadata'; +// Import the worker (handled by worker-loader) +import BackgroundSerializerWorker from './BackgroundSerializer.worker'; + +const gd: libGDevelop = global.gd; + +type BackgroundSerializerWorkerOutMessage = + | {| type: 'DONE', requestId: number, result: any |} + | {| type: 'ERROR', requestId: number, message: string |}; + +let serializerWorker: ?Worker = null; +let nextRequestId = 1; +const pendingRequests: Map< + number, + { resolve: (result: any) => void, reject: (error: Error) => void } +> = new Map(); + +const log = (message: string) => { + console.log(`[BackgroundSerializer] ${message}`); +}; + +const getOrCreateBackgroundSerializerWorker = (): Worker => { + if (serializerWorker) { + return serializerWorker; + } + + // $FlowExpectedError - worker-loader types aren't recognized by Flow + serializerWorker = new BackgroundSerializerWorker(); + + // Set up message handler + serializerWorker.onmessage = (event: MessageEvent) => { + const data: BackgroundSerializerWorkerOutMessage = (event.data: any); + + const pending = pendingRequests.get(data.requestId); + if (!pending) { + console.warn( + `[BackgroundSerializer] Received message for unknown request ID #${ + data.requestId + }.` + ); + return; + } + + if (data.type === 'DONE') { + pending.resolve(data.result); + return; + } + pending.reject(new Error(data.message || 'Unknown error')); + }; + + serializerWorker.onerror = error => { + console.error('[BackgroundSerializer] Worker sent an error.', error); + }; + + return serializerWorker; +}; + +const sendMessageToBackgroundSerializerWorker = (message: {| + type: 'SERIALIZE_TO_JSON' | 'SERIALIZE_TO_JS_OBJECT', + binary: Uint8Array, + versionWithHash: string, +|}) => { + const worker = getOrCreateBackgroundSerializerWorker(); + + return new Promise((resolve, reject) => { + const requestId = nextRequestId++; + pendingRequests.set(requestId, { resolve, reject }); + worker.postMessage({ ...message, requestId }); + }); +}; + +export async function serializeInBackground( + type: 'SERIALIZE_TO_JSON' | 'SERIALIZE_TO_JS_OBJECT', + serializable: gdSerializable +): Promise { + const startTime = Date.now(); + const serializedElement = new gd.SerializerElement(); + serializable.serializeTo(serializedElement); + + const serializeToEndTime = Date.now(); + + let binaryPtr = 0; + try { + binaryPtr = gd.BinarySerializer.createBinarySnapshot(serializedElement); + if (!binaryPtr) { + throw new Error('Failed to create binary snapshot.'); + } + + const binarySize = gd.BinarySerializer.getLastBinarySnapshotSize(); + const binaryView = new Uint8Array(gd.HEAPU8.buffer, binaryPtr, binarySize); + // Copy the buffer out of the WASM heap so it can be transferred. + const binaryBuffer = binaryView.slice(); + + const binaryBufferEndTime = Date.now(); + log( + `Spent ${binaryBufferEndTime - startTime}ms on main thread (including ${serializeToEndTime - startTime}ms for SerializerElement serialization and ${binaryBufferEndTime - serializeToEndTime}ms for BinaryBuffer preparation).` + ); + + const result = await sendMessageToBackgroundSerializerWorker({ + type, + binary: binaryBuffer, + versionWithHash: VersionMetadata.versionWithHash, + }); + + const workerPromiseEndTime = Date.now(); + log( + `The worker returned in ${workerPromiseEndTime - binaryBufferEndTime}ms.` + ); + + return result; + } finally { + if (binaryPtr) { + gd.BinarySerializer.freeBinarySnapshot(binaryPtr); + } + serializedElement.delete(); + } +} + +export const serializeToJSONInBackground = async ( + serializable: gdSerializable +): Promise => { + return serializeInBackground('SERIALIZE_TO_JSON', serializable); +}; + +export const serializeToJSObjectInBackground = async ( + serializable: gdSerializable +): Promise => { + return serializeInBackground('SERIALIZE_TO_JS_OBJECT', serializable); +}; diff --git a/newIDE/app/src/Utils/BackgroundSerializer.worker.js b/newIDE/app/src/Utils/BackgroundSerializer.worker.js new file mode 100644 index 000000000000..e503407d287c --- /dev/null +++ b/newIDE/app/src/Utils/BackgroundSerializer.worker.js @@ -0,0 +1,115 @@ +/* eslint-env worker */ +// @flow + +let modulePromise /*: ?Promise*/ = null; + +const log = (message /*: string */) => { + console.log(`[BackgroundSerializerWorker] ${message}`); +}; + +const getLibGDevelop = (versionWithHash /*: string */) => { + if (modulePromise) return modulePromise; + + modulePromise = new Promise((resolve, reject) => { + try { + const url = `/libGD.js?cache-buster=${versionWithHash}`; + // Load libGD.js in the worker context. + // eslint-disable-next-line no-undef + importScripts(url); + + // eslint-disable-next-line no-undef + // $FlowFixMe + if (typeof initializeGDevelopJs !== 'function') { + reject(new Error('Missing initializeGDevelopJs in worker')); + return; + } + + // eslint-disable-next-line no-undef + initializeGDevelopJs({ + // Override the resolved URL for the .wasm file, + // to ensure a new version is fetched when the version changes. + locateFile: (path /*: string */, prefix /*: string */) => { + // This function is called by Emscripten to locate the .wasm file only. + // As the wasm is at the root of the public folder, we can just return + // the path to the file. + // Plus, on Electron, the prefix seems to be pointing to the root of the + // app.asar archive, which is completely wrong. + return path + `?cache-buster=${versionWithHash}`; + }, + }) + .then(module => { + resolve(module); + }) + .catch(reject); + } catch (error) { + reject(error); + return; + } + }); + + return modulePromise; +}; + +const unserializeBinarySnapshotToJson = ( + gd /*: libGDevelop */, + binary /*: Uint8Array */ +) => { + const binaryArray = + binary instanceof Uint8Array ? binary : new Uint8Array(binary); + const binarySize = binaryArray.byteLength || binaryArray.length; + + // Allocate memory in Emscripten heap and copy binary data + const binaryPtr = gd._malloc(binarySize); + gd.HEAPU8.set(binaryArray, binaryPtr); + + const element = gd.BinarySerializer.deserializeBinarySnapshot( + binaryPtr, + binarySize + ); + + // Free the input buffer + gd._free(binaryPtr); + + if (element.ptr === 0) { + throw new Error('Failed to deserialize binary snapshot.'); + } + + const json = gd.Serializer.toJSON(element); + element.delete(); + return json; +}; + +// eslint-disable-next-line no-restricted-globals +self.onmessage = async (event /*: MessageEvent */) => { + // $FlowExpectedError + const { type, binary, requestId, versionWithHash } = event.data || {}; + + const startTime = Date.now(); + + log(`Request #${requestId} received (${type}).`); + if (type !== 'SERIALIZE_TO_JSON' && type !== 'SERIALIZE_TO_JS_OBJECT') return; + + try { + const gd = await getLibGDevelop(versionWithHash); + + const json = unserializeBinarySnapshotToJson(gd, binary); + const result = type === 'SERIALIZE_TO_JSON' ? json : JSON.parse(json); + + log(`Request #${requestId} done in ${Date.now() - startTime}ms.`); + + // eslint-disable-next-line no-restricted-globals + self.postMessage({ + type: 'DONE', + result, + requestId, + duration: Date.now() - startTime, + }); + } catch (error) { + // eslint-disable-next-line no-restricted-globals + self.postMessage({ + type: 'ERROR', + requestId, + message: error.message, + }); + } +}; diff --git a/newIDE/app/src/Utils/Serializer.js b/newIDE/app/src/Utils/Serializer.js index 997dcef403da..8f5cc6b3b8b6 100644 --- a/newIDE/app/src/Utils/Serializer.js +++ b/newIDE/app/src/Utils/Serializer.js @@ -1,9 +1,4 @@ // @flow - -import VersionMetadata from '../Version/VersionMetadata'; -// Import the worker (handled by worker-loader) -import SerializerWorker from './Serializer.worker'; - const gd: libGDevelop = global.gd; /** @@ -147,143 +142,6 @@ export function serializeToJSON( return json; } -type SerializerWorkerOutMessage = - | {| type: 'serialized', object: any |} - | {| type: 'error', message: string |}; - -let serializerWorker: ?Worker = null; - -const getOrCreateSerializerWorker = (): Worker => { - if (!serializerWorker) { - // $FlowExpectedError - worker-loader types aren't recognized by Flow - serializerWorker = new SerializerWorker(); - } - - return serializerWorker; -}; - -export function terminateSerializerWorkerForTests() { - if (serializerWorker) { - serializerWorker.terminate(); - serializerWorker = null; - } -} - -/** - * Serialize a gdSerializable into a JS object while keeping the slowest parts - * off the main thread. - */ -export async function serializeToJSObjectInBackground( - serializable: gdSerializable, - methodName: string = 'serializeTo' -): Promise { - const startTime = Date.now(); - const serializedElement = new gd.SerializerElement(); - serializable[methodName](serializedElement); - const serializeToTime = Date.now(); - console.log( - '[Main thread] serializeTo done in: ', - serializeToTime - startTime, - 'ms' - ); - - // Allocate space for the output size in WASM heap. - // $FlowExpectedError - accessing emscripten memory helpers. - const sizePtr = gd._malloc(4); - if (!sizePtr) { - serializedElement.delete(); - throw new Error('Failed to allocate memory for serialization size.'); - } - - let binaryPtr = 0; - try { - // $FlowExpectedError - accessing emscripten exports. - binaryPtr = gd._createBinarySnapshot(serializedElement.ptr, sizePtr); - if (!binaryPtr) { - throw new Error('Failed to create binary snapshot.'); - } - - const createBinarySnapshotTime = Date.now(); - console.log( - '[Main thread] createBinarySnapshot done in: ', - createBinarySnapshotTime - serializeToTime, - 'ms' - ); - - // $FlowExpectedError - accessing emscripten memory helpers. - const binarySize = gd.HEAPU32[sizePtr >> 2]; - // $FlowExpectedError - accessing emscripten memory helpers. - const binaryView = new Uint8Array(gd.HEAPU8.buffer, binaryPtr, binarySize); - // Copy the buffer out of the WASM heap so it can be transferred. - const binaryBuffer = binaryView.slice(); - - const binaryBufferTime = Date.now(); - console.log( - '[Main thread] BinaryBuffer copied/prepared in: ', - binaryBufferTime - createBinarySnapshotTime, - 'ms' - ); - - const worker = getOrCreateSerializerWorker(); - const cacheBuster = VersionMetadata.versionWithHash; - - const workerPromiseStartTime = Date.now(); - const object = await new Promise((resolve, reject) => { - const handleMessage = (event: MessageEvent) => { - const data: SerializerWorkerOutMessage = event.data; - if (data.type === 'serialized') { - worker.removeEventListener('message', handleMessage); - worker.removeEventListener('error', handleError); - resolve(data.object); - } else if (data.type === 'error') { - worker.removeEventListener('message', handleMessage); - worker.removeEventListener('error', handleError); - reject(new Error(data.message)); - } - }; - - const handleError = (error: any) => { - worker.removeEventListener('message', handleMessage); - worker.removeEventListener('error', handleError); - reject(error); - }; - - worker.addEventListener('message', handleMessage); - worker.addEventListener('error', handleError); - worker.postMessage( - { - type: 'serialize', - binary: binaryBuffer, - cacheBuster, - }, - [binaryBuffer.buffer] - ); - console.log( - '[Main thread] main thread work done in: ', - Date.now() - startTime, - 'ms' - ); - }); - - const workerPromiseTime = Date.now(); - console.log( - '[Main thread] The worker promise (doing JSON serialization from the binary buffer) returned in: ', - workerPromiseTime - workerPromiseStartTime, - 'ms' - ); - - return object; - } finally { - if (binaryPtr) { - // $FlowExpectedError - accessing emscripten exports. - gd._freeBinarySnapshot(binaryPtr); - } - // $FlowExpectedError - accessing emscripten exports. - gd._free(sizePtr); - serializedElement.delete(); - } -} - /** * Tool function to restore a serializable object from a JS object. * Most gd.* objects are "serializable", meaning they have a serializeTo diff --git a/newIDE/app/src/Utils/Serializer.worker.js b/newIDE/app/src/Utils/Serializer.worker.js deleted file mode 100644 index 0db922d17a54..000000000000 --- a/newIDE/app/src/Utils/Serializer.worker.js +++ /dev/null @@ -1,104 +0,0 @@ -/* eslint-env worker */ -// @flow - -let modulePromise /*: ?Promise*/ = null; - -const ensureLibGD = (cacheBuster /*: ?string*/) => { - if (modulePromise) return modulePromise; - - modulePromise = new Promise((resolve, reject) => { - try { - const libGDUrl = cacheBuster - ? `/libGD.js?cache-buster=${cacheBuster}` - : '/libGD.js'; - // Load libGD.js in the worker context. - // eslint-disable-next-line no-undef - importScripts(libGDUrl); - - // eslint-disable-next-line no-undef - if (typeof initializeGDevelopJs !== 'function') { - reject(new Error('Missing initializeGDevelopJs in worker')); - return; - } - - // eslint-disable-next-line no-undef - initializeGDevelopJs({ - locateFile: (path /*: string*/) => - cacheBuster ? `/${path}?cache-buster=${cacheBuster}` : `/${path}`, - }) - .then(module => { - resolve(module); - }) - .catch(reject); - } catch (error) { - reject(error); - return; - } - }); - - return modulePromise; -}; - -// eslint-disable-next-line no-restricted-globals -self.onmessage = async (event /*: MessageEvent*/) => { - const { type, binary, cacheBuster } = event.data || {}; - - if (type !== 'serialize') return; - - // TODO: handle request ids - - try { - console.log('Serializer worker: serialize started'); - const gd = await ensureLibGD(cacheBuster); - console.log('Serializer worker: libGD initialized'); - - const binaryArray = - binary instanceof Uint8Array ? binary : new Uint8Array(binary); - const binarySize = binaryArray.byteLength || binaryArray.length; - - const startTime = Date.now(); - - const binaryPtr = gd._malloc(binarySize); - gd.HEAPU8.set(binaryArray, binaryPtr); - - const elementPtr = gd._deserializeBinarySnapshot(binaryPtr, binarySize); - if (!elementPtr) { - gd._free(binaryPtr); - // eslint-disable-next-line no-restricted-globals - self.postMessage({ - type: 'error', - message: 'Failed to deserialize binary snapshot', - }); - return; - } - - const element = - typeof gd.wrapPointer === 'function' - ? gd.wrapPointer(elementPtr, gd.SerializerElement) - : new gd.SerializerElement(elementPtr); - - const json = gd.Serializer.toJSON(element); - const object = JSON.parse(json); - - gd._free(binaryPtr); - element.delete(); - - console.log( - 'Serializer worker: serialize finished in', - Date.now() - startTime, - 'ms' - ); - // eslint-disable-next-line no-restricted-globals - self.postMessage({ - type: 'serialized', - object, - duration: Date.now() - startTime, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-globals - self.postMessage({ - type: 'error', - message: error.message, - }); - } -}; From 726b34f7871493b9e9b0dce73f46108d7f56171b Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Mon, 22 Dec 2025 17:16:04 +0100 Subject: [PATCH 03/10] Fix formatting --- .../CloudStorageProvider/CloudProjectWriter.js | 4 +--- newIDE/app/src/Utils/BackgroundSerializer.js | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js index eb0ede3e3b79..6bdecf3c656b 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js @@ -12,9 +12,7 @@ import { } from '../../Utils/GDevelopServices/Project'; import type { $AxiosError } from 'axios'; import type { MessageDescriptor } from '../../Utils/i18n/MessageDescriptor.flow'; -import { - serializeToJSON, -} from '../../Utils/Serializer'; +import { serializeToJSON } from '../../Utils/Serializer'; import { serializeToJSONInBackground } from '../../Utils/BackgroundSerializer'; import { t } from '@lingui/macro'; import { diff --git a/newIDE/app/src/Utils/BackgroundSerializer.js b/newIDE/app/src/Utils/BackgroundSerializer.js index 5d72dcded021..8ab9f34f6a45 100644 --- a/newIDE/app/src/Utils/BackgroundSerializer.js +++ b/newIDE/app/src/Utils/BackgroundSerializer.js @@ -95,7 +95,10 @@ export async function serializeInBackground( const binaryBufferEndTime = Date.now(); log( - `Spent ${binaryBufferEndTime - startTime}ms on main thread (including ${serializeToEndTime - startTime}ms for SerializerElement serialization and ${binaryBufferEndTime - serializeToEndTime}ms for BinaryBuffer preparation).` + `Spent ${binaryBufferEndTime - + startTime}ms on main thread (including ${serializeToEndTime - + startTime}ms for SerializerElement serialization and ${binaryBufferEndTime - + serializeToEndTime}ms for BinaryBuffer preparation).` ); const result = await sendMessageToBackgroundSerializerWorker({ From bba9b8c2bd5615cf4cd2231eadbe11b77b55e772 Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Tue, 23 Dec 2025 14:21:24 +0100 Subject: [PATCH 04/10] Add tests --- .../GDCore/Serialization/BinarySerializer.cpp | 12 +- GDevelop.js/__tests__/Serializer.js | 221 ++++++++++++------ newIDE/app/src/Utils/BackgroundSerializer.js | 71 +++--- 3 files changed, 189 insertions(+), 115 deletions(-) diff --git a/Core/GDCore/Serialization/BinarySerializer.cpp b/Core/GDCore/Serialization/BinarySerializer.cpp index 81e557fae03c..68ebffd49a9c 100644 --- a/Core/GDCore/Serialization/BinarySerializer.cpp +++ b/Core/GDCore/Serialization/BinarySerializer.cpp @@ -5,7 +5,7 @@ */ #include "GDCore/Serialization/BinarySerializer.h" - +#include "GDCore/Tools/Log.h" #include #include @@ -100,12 +100,14 @@ bool BinarySerializer::DeserializeFromBinaryBuffer(const uint8_t* buffer, // Read and verify magic header uint32_t magic; if (!Read(ptr, end, magic) || magic != 0x47444253) { + gd::LogError("Failed to deserialize binary snapshot: invalid magic."); return false; // Invalid magic } // Read version uint32_t version; if (!Read(ptr, end, version) || version != 1) { + gd::LogError("Failed to deserialize binary snapshot: unsupported version."); return false; // Unsupported version } @@ -118,12 +120,16 @@ bool BinarySerializer::DeserializeElement(const uint8_t*& ptr, SerializerElement& element) { NodeType nodeType; if (!Read(ptr, end, nodeType) || nodeType != NodeType::Element) { + gd::LogError("Failed to deserialize binary snapshot: invalid node type."); return false; } // Deserialize value NodeType valueType; - if (!Read(ptr, end, valueType)) return false; + if (!Read(ptr, end, valueType)) { + gd::LogError("Failed to deserialize binary snapshot: invalid value type."); + return false; + } if (valueType != NodeType::ValueUndefined) { SerializerValue value; @@ -266,6 +272,7 @@ void BinarySerializer::FreeBinarySnapshot(uintptr_t bufferPtr) { SerializerElement* BinarySerializer::DeserializeBinarySnapshot(uintptr_t bufferPtr, size_t size) { if (!bufferPtr || size == 0) { + gd::LogError("Failed to deserialize binary snapshot: invalid buffer pointer or size."); return nullptr; } @@ -273,6 +280,7 @@ SerializerElement* BinarySerializer::DeserializeBinarySnapshot(uintptr_t bufferP SerializerElement* element = new SerializerElement(); if (!DeserializeFromBinaryBuffer(buffer, size, *element)) { + gd::LogError("Failed to deserialize binary snapshot."); delete element; return nullptr; } diff --git a/GDevelop.js/__tests__/Serializer.js b/GDevelop.js/__tests__/Serializer.js index deef304218bd..d0a72e1b0f06 100644 --- a/GDevelop.js/__tests__/Serializer.js +++ b/GDevelop.js/__tests__/Serializer.js @@ -127,80 +127,149 @@ describe('libGD.js object serialization', function () { }); }); - // describe('gd.createBinarySnapshot and gd.deserializeBinarySnapshot', function () { - // const checkBinaryRoundTrip = (json) => { - // // Create a SerializerElement from JSON - // const originalElement = gd.Serializer.fromJSON(json); - - // // Create binary snapshot - // const binaryBuffer = gd.createBinarySnapshot(originalElement); - // expect(binaryBuffer).toBeInstanceOf(Uint8Array); - // expect(binaryBuffer.length).toBeGreaterThan(0); - - // // Deserialize binary snapshot - // const restoredElement = gd.deserializeBinarySnapshot(binaryBuffer); - - // // Convert back to JSON and compare - // const outputJson = gd.Serializer.toJSON(restoredElement); - - // restoredElement.delete(); - // originalElement.delete(); - - // expect(outputJson).toBe(json); - // }; - - // it('should round-trip simple values', function () { - // checkBinaryRoundTrip('"hello"'); - // checkBinaryRoundTrip('123'); - // checkBinaryRoundTrip('123.456'); - // checkBinaryRoundTrip('true'); - // checkBinaryRoundTrip('false'); - // }); - - // it('should round-trip strings with unicode characters', function () { - // checkBinaryRoundTrip('"String with 官话 characters"'); - // checkBinaryRoundTrip('"Émojis: 🎮🎲🎯"'); - // }); - - // it('should round-trip objects', function () { - // checkBinaryRoundTrip('{}'); - // checkBinaryRoundTrip('{"a":"b"}'); - // checkBinaryRoundTrip('{"a":{"nested":"value"}}'); - // checkBinaryRoundTrip( - // '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3.0}}' - // ); - // }); - - // it('should round-trip arrays', function () { - // checkBinaryRoundTrip('[]'); - // checkBinaryRoundTrip('[1]'); - // checkBinaryRoundTrip('[1,2,3]'); - // checkBinaryRoundTrip('[{"a":1},{"b":2}]'); - // checkBinaryRoundTrip('{"items":[1,2,3],"nested":[{"x":1},{"y":2}]}'); - // }); - - // it('should round-trip a complex object like a Text Object', function () { - // var obj = new gd.TextObject('testObject'); - // obj.setText('Text with 官话 characters'); - - // var serializedElement = new gd.SerializerElement(); - // obj.serializeTo(serializedElement); - - // // Create binary snapshot - // const binaryBuffer = gd.createBinarySnapshot(serializedElement); - - // // Deserialize binary snapshot - // const restoredElement = gd.deserializeBinarySnapshot(binaryBuffer); - - // // Compare JSON output - // const originalJson = gd.Serializer.toJSON(serializedElement); - // const restoredJson = gd.Serializer.toJSON(restoredElement); - - // expect(restoredJson).toBe(originalJson); - - // restoredElement.delete(); - // serializedElement.delete(); - // obj.delete(); - // }); - // }); + describe('gd.BinarySerializer', function () { + const serializeToBinarySnapshot = (serializerElement) => { + // Create binary snapshot + const binaryPtr = + gd.BinarySerializer.createBinarySnapshot(serializerElement); + const binarySize = gd.BinarySerializer.getLastBinarySnapshotSize(); + + if (!binaryPtr) { + throw new Error('Failed to create binary snapshot.'); + } + + const binaryView = new Uint8Array( + gd.HEAPU8.buffer, + binaryPtr, + binarySize + ); + // Copy the buffer out of the WASM heap to simulate it was transferred. + const binaryBuffer = binaryView.slice(); + + gd.BinarySerializer.freeBinarySnapshot(binaryPtr); + + return binaryBuffer; + }; + + const serializeJsonToBinarySnapshot = (json) => { + const element = gd.Serializer.fromJSON(json); + // Do NOT delete element, it's a static value. + + return serializeToBinarySnapshot(element); + }; + + const unserializeBinarySnapshotToJson = (binaryBuffer) => { + const binaryArray = + binaryBuffer instanceof Uint8Array + ? binaryBuffer + : new Uint8Array(binaryBuffer); + const binarySize = binaryArray.byteLength || binaryArray.length; + + // Allocate memory in Emscripten heap and copy binary data + const binaryPtr = gd._malloc(binarySize); + gd.HEAPU8.set(binaryArray, binaryPtr); + + const element = gd.BinarySerializer.deserializeBinarySnapshot( + binaryPtr, + binarySize + ); + + // Free the input buffer + gd._free(binaryPtr); + + if (element.ptr === 0) { + throw new Error('Failed to deserialize binary snapshot.'); + } + + const json = gd.Serializer.toJSON(element); + element.delete(); + return json; + }; + + const checkBinaryRoundTrip = (json) => { + const binaryBuffer = serializeJsonToBinarySnapshot(json); + expect(binaryBuffer).toBeInstanceOf(Uint8Array); + expect(binaryBuffer.length).toBeGreaterThan(0); + + const outputJson = unserializeBinarySnapshotToJson(binaryBuffer); + expect(outputJson).toBe(json); + }; + + it('should round-trip simple values', function () { + checkBinaryRoundTrip('"hello"'); + checkBinaryRoundTrip('"hello"'); + checkBinaryRoundTrip('123'); + checkBinaryRoundTrip('123.456'); + checkBinaryRoundTrip('true'); + checkBinaryRoundTrip('false'); + }); + + it('should round-trip strings with unicode characters', function () { + checkBinaryRoundTrip('"String with 官话 characters"'); + checkBinaryRoundTrip('"Émojis: 🎮🎲🎯"'); + }); + + it('should round-trip objects', function () { + checkBinaryRoundTrip('{}'); + checkBinaryRoundTrip('{"a":"b"}'); + checkBinaryRoundTrip('{"a":{"nested":"value"}}'); + checkBinaryRoundTrip( + '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3.0}}' + ); + }); + + it('should round-trip arrays', function () { + checkBinaryRoundTrip('[]'); + checkBinaryRoundTrip('[1]'); + checkBinaryRoundTrip('[1,2,3]'); + checkBinaryRoundTrip('[{"a":1},{"b":2}]'); + checkBinaryRoundTrip('{"items":[1,2,3],"nested":[{"x":1},{"y":2}]}'); + }); + + it('should round-trip a complex object like a Text Object', function () { + const obj = new gd.TextObject('testObject'); + obj.setText('Text with 官话 characters'); + + const serializedElement = new gd.SerializerElement(); + obj.serializeTo(serializedElement); + + const binaryBuffer = serializeToBinarySnapshot(serializedElement); + const jsonFromBinaryBuffer = + unserializeBinarySnapshotToJson(binaryBuffer); + + // Compare JSON output from the original SerializerElement + // with the JSON obtained from the binary buffer: + const originalJson = gd.Serializer.toJSON(serializedElement); + serializedElement.delete(); + obj.delete(); + + expect(jsonFromBinaryBuffer).toBe(originalJson); + }); + + it('should round-trip a complex object like a Project', function () { + const project = new gd.ProjectHelper.createNewGDJSProject(); + const layout = project.insertNewLayout('Scene', 0); + layout.getObjects().insertNewObject(project, 'Sprite', 'Object1', 0); + layout.getObjects().insertNewObject(project, 'Sprite', 'Object2', 1); + const instance1 = layout.getInitialInstances().insertNewInitialInstance(); + const instance2 = layout.getInitialInstances().insertNewInitialInstance(); + instance1.setObjectName('Object1'); + instance2.setObjectName('Object2'); + + const serializedElement = new gd.SerializerElement(); + project.serializeTo(serializedElement); + + const binaryBuffer = serializeToBinarySnapshot(serializedElement); + const jsonFromBinaryBuffer = + unserializeBinarySnapshotToJson(binaryBuffer); + + // Compare JSON output from the original SerializerElement + // with the JSON obtained from the binary buffer: + const originalJson = gd.Serializer.toJSON(serializedElement); + serializedElement.delete(); + project.delete(); + + expect(jsonFromBinaryBuffer).toBe(originalJson); + }); + }); }); diff --git a/newIDE/app/src/Utils/BackgroundSerializer.js b/newIDE/app/src/Utils/BackgroundSerializer.js index 8ab9f34f6a45..60afccff4f09 100644 --- a/newIDE/app/src/Utils/BackgroundSerializer.js +++ b/newIDE/app/src/Utils/BackgroundSerializer.js @@ -76,49 +76,46 @@ export async function serializeInBackground( serializable: gdSerializable ): Promise { const startTime = Date.now(); - const serializedElement = new gd.SerializerElement(); + let serializedElement = new gd.SerializerElement(); serializable.serializeTo(serializedElement); const serializeToEndTime = Date.now(); - let binaryPtr = 0; - try { - binaryPtr = gd.BinarySerializer.createBinarySnapshot(serializedElement); - if (!binaryPtr) { - throw new Error('Failed to create binary snapshot.'); - } + const binaryPtr = gd.BinarySerializer.createBinarySnapshot(serializedElement); + const binarySize = gd.BinarySerializer.getLastBinarySnapshotSize(); + serializedElement.delete(); + serializedElement = null; - const binarySize = gd.BinarySerializer.getLastBinarySnapshotSize(); - const binaryView = new Uint8Array(gd.HEAPU8.buffer, binaryPtr, binarySize); - // Copy the buffer out of the WASM heap so it can be transferred. - const binaryBuffer = binaryView.slice(); - - const binaryBufferEndTime = Date.now(); - log( - `Spent ${binaryBufferEndTime - - startTime}ms on main thread (including ${serializeToEndTime - - startTime}ms for SerializerElement serialization and ${binaryBufferEndTime - - serializeToEndTime}ms for BinaryBuffer preparation).` - ); - - const result = await sendMessageToBackgroundSerializerWorker({ - type, - binary: binaryBuffer, - versionWithHash: VersionMetadata.versionWithHash, - }); - - const workerPromiseEndTime = Date.now(); - log( - `The worker returned in ${workerPromiseEndTime - binaryBufferEndTime}ms.` - ); - - return result; - } finally { - if (binaryPtr) { - gd.BinarySerializer.freeBinarySnapshot(binaryPtr); - } - serializedElement.delete(); + if (!binaryPtr) { + throw new Error('Failed to create binary snapshot.'); } + + const binaryView = new Uint8Array(gd.HEAPU8.buffer, binaryPtr, binarySize); + // Copy the buffer out of the WASM heap so it can be transferred. + const binaryBuffer = binaryView.slice(); + + gd.BinarySerializer.freeBinarySnapshot(binaryPtr); + + const binaryBufferEndTime = Date.now(); + log( + `Spent ${binaryBufferEndTime - + startTime}ms on main thread (including ${serializeToEndTime - + startTime}ms for SerializerElement serialization and ${binaryBufferEndTime - + serializeToEndTime}ms for BinaryBuffer preparation).` + ); + + const result = await sendMessageToBackgroundSerializerWorker({ + type, + binary: binaryBuffer, + versionWithHash: VersionMetadata.versionWithHash, + }); + + const workerPromiseEndTime = Date.now(); + log( + `The worker returned in ${workerPromiseEndTime - binaryBufferEndTime}ms.` + ); + + return result; } export const serializeToJSONInBackground = async ( From 10d16b97c98f92a028a2171ab61d7e84fba3e1d4 Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Tue, 23 Dec 2025 14:31:28 +0100 Subject: [PATCH 05/10] Fix typing --- GDevelop.js/Bindings/Bindings.idl | 16 ++++++++-------- GDevelop.js/types.d.ts | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/GDevelop.js/Bindings/Bindings.idl b/GDevelop.js/Bindings/Bindings.idl index 108dc555a4c9..1dda800ed39e 100644 --- a/GDevelop.js/Bindings/Bindings.idl +++ b/GDevelop.js/Bindings/Bindings.idl @@ -1605,17 +1605,17 @@ interface Serializer { [NoDelete] interface BinarySerializer { - // Create a binary snapshot, returns pointer to buffer in Emscripten heap - static unsigned long STATIC_CreateBinarySnapshot([Ref] SerializerElement element); + // Create a binary snapshot, returns pointer to buffer in Emscripten heap + unsigned long STATIC_CreateBinarySnapshot([Ref] SerializerElement element); - // Get the size of the last created snapshot - static unsigned long STATIC_GetLastBinarySnapshotSize(); + // Get the size of the last created snapshot + unsigned long STATIC_GetLastBinarySnapshotSize(); - // Free the binary snapshot - static void STATIC_FreeBinarySnapshot(unsigned long bufferPtr); + // Free the binary snapshot + void STATIC_FreeBinarySnapshot(unsigned long bufferPtr); - // Deserialize from a pointer in Emscripten heap - static SerializerElement STATIC_DeserializeBinarySnapshot(unsigned long bufferPtr, unsigned long size); + // Deserialize from a pointer in Emscripten heap + SerializerElement STATIC_DeserializeBinarySnapshot(unsigned long bufferPtr, unsigned long size); }; interface ObjectAssetSerializer { diff --git a/GDevelop.js/types.d.ts b/GDevelop.js/types.d.ts index 93a5a764ae7f..dcb868473aa6 100644 --- a/GDevelop.js/types.d.ts +++ b/GDevelop.js/types.d.ts @@ -1326,10 +1326,10 @@ export class Serializer extends EmscriptenObject { } export class BinarySerializer extends EmscriptenObject { - static unsigned long CreateBinarySnapshot(element: SerializerElement): static; - static unsigned long GetLastBinarySnapshotSize(): static; - static void FreeBinarySnapshot(bufferPtr: number): static; - static serializerElement DeserializeBinarySnapshot(bufferPtr: number, size: number): static; + static createBinarySnapshot(element: SerializerElement): number; + static getLastBinarySnapshotSize(): number; + static freeBinarySnapshot(bufferPtr: number): void; + static deserializeBinarySnapshot(bufferPtr: number, size: number): SerializerElement; } export class ObjectAssetSerializer extends EmscriptenObject { From d48a81b34b3ce7cd9536950fa536b5aac5af11e7 Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Tue, 23 Dec 2025 15:01:28 +0100 Subject: [PATCH 06/10] Fix tests --- .../CloudStoageProviderInternalName.js | 3 ++ .../CloudStorageProvider/index.js | 3 +- .../LocalFileStorageProviderInternalName.js | 3 ++ .../LocalFileStorageProvider/index.js | 3 +- .../ResourceMover/LocalResourceMover.js | 45 ++++++++++--------- 5 files changed, 35 insertions(+), 22 deletions(-) create mode 100644 newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudStoageProviderInternalName.js create mode 100644 newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalFileStorageProviderInternalName.js diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudStoageProviderInternalName.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudStoageProviderInternalName.js new file mode 100644 index 000000000000..fde3f9742471 --- /dev/null +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudStoageProviderInternalName.js @@ -0,0 +1,3 @@ +// @flow + +export const cloudStorageProviderInternalName = 'Cloud'; diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js index eb3321546811..e6cdbb7b5c1f 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js @@ -25,6 +25,7 @@ import { } from './CloudProjectOpener'; import Cloud from '../../UI/CustomSvgIcons/Cloud'; import { generateGetResourceActions } from './CloudProjectResourcesHandler'; +import { cloudStorageProviderInternalName } from './CloudStoageProviderInternalName'; const isURL = (filename: string) => { return ( @@ -37,7 +38,7 @@ const isURL = (filename: string) => { }; export default ({ - internalName: 'Cloud', + internalName: cloudStorageProviderInternalName, name: t`GDevelop Cloud`, renderIcon: props => , hiddenInOpenDialog: true, diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalFileStorageProviderInternalName.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalFileStorageProviderInternalName.js new file mode 100644 index 000000000000..1eee2a30afa7 --- /dev/null +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalFileStorageProviderInternalName.js @@ -0,0 +1,3 @@ +// @flow + +export const localFileStorageProviderInternalName = 'LocalFile'; diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js index a82f98bed3e6..31606e8c9aba 100644 --- a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js @@ -33,13 +33,14 @@ import { } from './LocalProjectResourcesHandler'; import { allResourceKindsAndMetadata } from '../../ResourcesList/ResourceSource'; import { setupResourcesWatcher } from './LocalFileResourcesWatcher'; +import { localFileStorageProviderInternalName } from './LocalFileStorageProviderInternalName'; /** * Use the Electron APIs to provide access to the native * file system (with native save/open dialogs). */ export default ({ - internalName: 'LocalFile', + internalName: localFileStorageProviderInternalName, name: t`Your computer`, renderIcon: props => , getFileMetadataFromAppArguments: (appArguments: AppArguments) => { diff --git a/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js b/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js index 56b9beb49879..93acc9b31148 100644 --- a/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js +++ b/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js @@ -4,10 +4,8 @@ import { type MoveAllProjectResourcesResult, type MoveAllProjectResourcesFunction, } from './index'; -import LocalFileStorageProvider from '../LocalFileStorageProvider'; import { moveUrlResourcesToLocalFiles } from '../LocalFileStorageProvider/LocalFileResourceMover'; import UrlStorageProvider from '../UrlStorageProvider'; -import CloudStorageProvider from '../CloudStorageProvider'; import LocalFileSystem from '../../ExportAndShare/LocalExporters/LocalFileSystem'; import assignIn from 'lodash/assignIn'; import optionalRequire from '../../Utils/OptionalRequire'; @@ -21,6 +19,8 @@ import { import { processByChunk } from '../../Utils/ProcessByChunk'; import { readLocalFileToFile } from '../../Utils/LocalFileUploader'; import { isURL, isBlobURL } from '../../ResourcesList/ResourceUtils'; +import { cloudStorageProviderInternalName } from '../CloudStorageProvider/CloudStoageProviderInternalName'; +import { localFileStorageProviderInternalName } from '../LocalFileStorageProvider/LocalFileStorageProviderInternalName'; const path = optionalRequire('path'); const gd: libGDevelop = global.gd; @@ -164,9 +164,10 @@ export const moveAllLocalResourcesToCloudResources = async ({ const movers: { [string]: MoveAllProjectResourcesFunction, } = { - [`${LocalFileStorageProvider.internalName}=>${ - LocalFileStorageProvider.internalName - }`]: async ({ project, newFileMetadata }: MoveAllProjectResourcesOptions) => { + [`${localFileStorageProviderInternalName}=>${localFileStorageProviderInternalName}`]: async ({ + project, + newFileMetadata, + }: MoveAllProjectResourcesOptions) => { // TODO: Ideally, errors while copying resources should be reported. // TODO: Report progress. const projectPath = path.dirname(newFileMetadata.fileIdentifier); @@ -189,9 +190,12 @@ const movers: { }, // When saving a Cloud project locally, all resources are downloaded (including // the ones on GDevelop Cloud or private game templates). - [`${CloudStorageProvider.internalName}=>${ - LocalFileStorageProvider.internalName - }`]: ({ project, newFileMetadata, onProgress, authenticatedUser }) => + [`${cloudStorageProviderInternalName}=>${localFileStorageProviderInternalName}`]: ({ + project, + newFileMetadata, + onProgress, + authenticatedUser, + }) => moveUrlResourcesToLocalFiles({ project, fileMetadata: newFileMetadata, @@ -204,9 +208,14 @@ const movers: { // for resources). // This is also helpful to download private game templates resources so that // the game can be opened offline. - [`${UrlStorageProvider.internalName}=>${ - LocalFileStorageProvider.internalName - }`]: ({ project, newFileMetadata, onProgress, authenticatedUser }) => + [`${ + UrlStorageProvider.internalName + }=>${localFileStorageProviderInternalName}`]: ({ + project, + newFileMetadata, + onProgress, + authenticatedUser, + }) => moveUrlResourcesToLocalFiles({ project, fileMetadata: newFileMetadata, @@ -217,20 +226,16 @@ const movers: { // Moving to GDevelop "Cloud" storage: // From a local project to a Cloud project, all resources are uploaded. - [`${LocalFileStorageProvider.internalName}=>${ - CloudStorageProvider.internalName - }`]: moveAllLocalResourcesToCloudResources, + [`${localFileStorageProviderInternalName}=>${cloudStorageProviderInternalName}`]: moveAllLocalResourcesToCloudResources, // From a Cloud project to another, resources need to be copied // (unless they are public URLs). - [`${CloudStorageProvider.internalName}=>${ - CloudStorageProvider.internalName - }`]: moveUrlResourcesToCloudProject, + [`${cloudStorageProviderInternalName}=>${cloudStorageProviderInternalName}`]: moveUrlResourcesToCloudProject, // Nothing to move around when going from a project on a public URL // to a cloud project (we could offer an option one day though to download // and upload the URL resources on GDevelop Cloud). - [`${UrlStorageProvider.internalName}=>${ - CloudStorageProvider.internalName - }`]: moveUrlResourcesToCloudProject, + [`${ + UrlStorageProvider.internalName + }=>${cloudStorageProviderInternalName}`]: moveUrlResourcesToCloudProject, }; const LocalResourceMover = { From b25a32ff3f9e573516e9433ee548ac9e3db04b5e Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Tue, 23 Dec 2025 16:08:00 +0100 Subject: [PATCH 07/10] Improve logs --- newIDE/app/src/MainFrame/index.js | 13 +-- .../CloudProjectWriter.js | 64 ++++++------- .../LocalProjectWriter.js | 90 +++++++++---------- newIDE/app/src/ProjectsStorage/index.js | 8 +- newIDE/app/src/Utils/Serializer.js | 34 ------- 5 files changed, 84 insertions(+), 125 deletions(-) diff --git a/newIDE/app/src/MainFrame/index.js b/newIDE/app/src/MainFrame/index.js index 8598fb7f2ccb..1f143128d966 100644 --- a/newIDE/app/src/MainFrame/index.js +++ b/newIDE/app/src/MainFrame/index.js @@ -109,6 +109,7 @@ import { type SaveAsOptions, type FileMetadataAndStorageProviderName, type ResourcesActionsMenuBuilder, + type SaveProjectOptions, } from '../ProjectsStorage'; import OpenFromStorageProviderDialog from '../ProjectsStorage/OpenFromStorageProviderDialog'; import SaveToStorageProviderDialog from '../ProjectsStorage/SaveToStorageProviderDialog'; @@ -3907,7 +3908,6 @@ const MainFrame = (props: Props) => { const saveProject = React.useCallback( async () => { - console.log('saveProject started'); if (!currentProject) return; // Prevent saving if there are errors in the extension modules, as // this can lead to corrupted projects. @@ -3924,19 +3924,13 @@ const MainFrame = (props: Props) => { return; } - console.log('getStorageProviderOperations call'); const storageProviderOperations = getStorageProviderOperations(); - const { - onSaveProject, - // canFileMetadataBeSafelySaved, - } = storageProviderOperations; + const { onSaveProject } = storageProviderOperations; if (!onSaveProject) { return saveProjectAs(); } - console.log('save ui settings'); saveUiSettings(state.editorTabs); - console.log('save ui settings done'); // Protect against concurrent saves, which can trigger issues with the // file system. @@ -3952,7 +3946,6 @@ const MainFrame = (props: Props) => { }); if (!shouldRestoreCheckedOutVersion) return; } - console.log('show message'); _showSnackMessage(i18n._(t`Saving...`), null); setIsSavingProject(true); @@ -3965,7 +3958,7 @@ const MainFrame = (props: Props) => { // store their values in variables now. const storageProviderInternalName = getStorageProvider().internalName; - const saveOptions = {}; + const saveOptions: SaveProjectOptions = {}; if (cloudProjectRecoveryOpenedVersionId) { saveOptions.previousVersion = cloudProjectRecoveryOpenedVersionId; } else { diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js index 6bdecf3c656b..e1a99511b9b7 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js @@ -1,7 +1,12 @@ // @flow import * as React from 'react'; import { type AuthenticatedUser } from '../../Profile/AuthenticatedUserContext'; -import { type FileMetadata, type SaveAsLocation, type SaveAsOptions } from '..'; +import { + type FileMetadata, + type SaveAsLocation, + type SaveAsOptions, + type SaveProjectOptions, +} from '..'; import { CLOUD_PROJECT_NAME_MAX_LENGTH, commitVersion, @@ -35,40 +40,36 @@ import { import { format } from 'date-fns'; import { getUserPublicProfile } from '../../Utils/GDevelopServices/User'; -const zipProject = async (project: gdProject): Promise<[Blob, string]> => { - console.log('--- zipProject started'); +const zipProject = async ({ + project, + useBackgroundSerializer, +}: { + project: gdProject, + useBackgroundSerializer: boolean, +}): Promise<{ zippedProject: Blob, projectJson: string }> => { const startTime = Date.now(); - const projectJson = serializeToJSON(project); - const serializeToJSONTime = Date.now(); - console.log( - '--- serializeToJSON done in: ', - serializeToJSONTime - startTime, - 'ms' - ); - const startTime2 = Date.now(); - // TODO: should serialize to JSON instead of JS object. - const projectJson2 = await serializeToJSONInBackground(project); - console.log( - '--- serializeToJSONInBackground done in: ', - Date.now() - startTime2, - 'ms (in total, including worker promise)' - ); - if (projectJson2 !== projectJson) { - console.log('Project JSONs are different.', projectJson, projectJson2); + let projectJson: string; + if (useBackgroundSerializer) { + projectJson = await serializeToJSONInBackground(project); + } else { + projectJson = serializeToJSON(project); } - const zipStartTime = Date.now(); + const serializeToJSONEndTime = Date.now(); + const zippedProject = await createZipWithSingleTextFile( projectJson, 'game.json' ); + console.log( - '[Main thread] Zipping done in: ', - Date.now() - zipStartTime, - 'ms' + `[CloudProjectWriter] Zipping done in ${Date.now() - + startTime}ms (including ${serializeToJSONEndTime - startTime}ms for ${ + useBackgroundSerializer ? 'background' : 'main' + } thread serialization).` ); - return [zippedProject, projectJson]; + return { zippedProject, projectJson }; }; const checkZipContent = async ( @@ -95,14 +96,17 @@ const zipAndPrepareProjectVersionForCommit = async ({ authenticatedUser: AuthenticatedUser, project: gdProject, cloudProjectId: string, - options?: {| previousVersion?: string, restoredFromVersionId?: string |}, + options?: SaveProjectOptions, |}): Promise<{| presignedUrl: string, zippedProject: Blob, |}> => { - const [presignedUrl, [zippedProject, projectJson]] = await Promise.all([ + const [presignedUrl, { zippedProject, projectJson }] = await Promise.all([ getPresignedUrlForVersionUpload(authenticatedUser, cloudProjectId), - zipProject(project), + zipProject({ + project, + useBackgroundSerializer: !!options && !!options.useBackgroundSerializer, + }), ]); const archiveIsSane = await checkZipContent(zippedProject, projectJson); @@ -129,7 +133,7 @@ const commitProjectVersion = async ({ presignedUrl: string, zippedProject: Blob, cloudProjectId: string, - options?: {| previousVersion?: string, restoredFromVersionId?: string |}, + options?: SaveProjectOptions, |}): Promise => { const newVersion = await retryIfFailed({ times: 2 }, () => commitVersion({ @@ -149,7 +153,7 @@ export const generateOnSaveProject = ( ) => async ( project: gdProject, fileMetadata: FileMetadata, - options?: {| previousVersion?: string, restoredFromVersionId?: string |}, + options?: SaveProjectOptions, actions: {| showAlert: ShowAlertFunction, showConfirmation: ShowConfirmFunction, diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js index 532d056093d0..579dc915a634 100644 --- a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js @@ -7,6 +7,7 @@ import { type FileMetadata, type SaveAsLocation, type SaveAsOptions, + type SaveProjectOptions, } from '../index'; import optionalRequire from '../../Utils/OptionalRequire'; import { @@ -107,46 +108,26 @@ const writeAndCheckFormattedJSONFile = async ( await writeAndCheckFile(content, filePath); }; -const writeProjectFiles = async ( +const writeProjectFiles = async ({ + project, + filePath, + projectPath, + useBackgroundSerializer, +}: { project: gdProject, filePath: string, - projectPath: string -): Promise => { - console.log('--- writeProjectFiles started'); - - const startTime2 = Date.now(); - const serializedProjectObject = serializeToJSObject(project); - console.log( - '--- serializeToJSObject done in: ', - Date.now() - startTime2, - 'ms (all on the main thread)' - ); - - try { - const startTime = Date.now(); - const serializedProjectObject2 = await serializeToJSObjectInBackground( - project - ); - console.log( - '--- serializeToJSObjectInBackground done in: ', - Date.now() - startTime, - 'ms (in total, including worker promise)' - ); - - if ( - JSON.stringify(serializedProjectObject) !== - JSON.stringify(serializedProjectObject2) - ) { - console.log( - 'Project JSONs are different.', - serializedProjectObject, - serializedProjectObject2 - ); - } - } catch (error) { - console.error('Unable to serialize to JS object in background:', error); - throw error; + projectPath: string, + useBackgroundSerializer: boolean, +}): Promise => { + const startTime = Date.now(); + + let serializedProjectObject; + if (useBackgroundSerializer) { + serializedProjectObject = await serializeToJSObjectInBackground(project); + } else { + serializedProjectObject = serializeToJSObject(project); } + const serializeEndTime = Date.now(); if (project.isFolderProject()) { const partialObjects = split(serializedProjectObject, { @@ -180,23 +161,21 @@ const writeProjectFiles = async ( }); }); } else { - return writeAndCheckFormattedJSONFile( - serializedProjectObject, - filePath - ).catch(err => { - console.error('Unable to write the project:', err); - throw err; - }); + await writeAndCheckFormattedJSONFile(serializedProjectObject, filePath); } + + console.log( + `[LocalProjectWriter] Project file(s) written in ${Date.now() - + startTime}ms (including ${serializeEndTime - startTime}ms for ${ + useBackgroundSerializer ? 'background' : 'main' + } thread serialization)` + ); }; export const onSaveProject = async ( project: gdProject, fileMetadata: FileMetadata, - unusedSaveOptions?: {| - previousVersion?: string, - restoredFromVersionId?: string, - |}, + saveOptions?: SaveProjectOptions, actions: {| showAlert: ShowAlertFunction, showConfirmation: ShowConfirmFunction, @@ -234,7 +213,13 @@ export const onSaveProject = async ( console.warn('Unable to clean project folder before saving project: ', e); } - await writeProjectFiles(project, filePath, projectPath); + await writeProjectFiles({ + project, + filePath, + projectPath, + useBackgroundSerializer: + !!saveOptions && !!saveOptions.useBackgroundSerializer, + }); return { wasSaved: true, fileMetadata: newFileMetadata, @@ -351,7 +336,12 @@ export const onSaveProjectAs = async ( const projectPath = path.dirname(filePath); project.setProjectFile(filePath); - await writeProjectFiles(project, filePath, projectPath); + await writeProjectFiles({ + project, + filePath, + projectPath, + useBackgroundSerializer: false, + }); return { wasSaved: true, fileMetadata: newFileMetadata, diff --git a/newIDE/app/src/ProjectsStorage/index.js b/newIDE/app/src/ProjectsStorage/index.js index e2590f913ece..d535823084aa 100644 --- a/newIDE/app/src/ProjectsStorage/index.js +++ b/newIDE/app/src/ProjectsStorage/index.js @@ -50,6 +50,12 @@ export type SaveAsLocation = {| // a new location where to save a project to. |}; +export type SaveProjectOptions = {| + previousVersion?: string, + restoredFromVersionId?: string, + useBackgroundSerializer?: boolean, +|}; + export type SaveAsOptions = {| generateNewProjectUuid?: boolean, setProjectNameFromLocation?: boolean, @@ -106,7 +112,7 @@ export type StorageProviderOperations = {| onSaveProject?: ( project: gdProject, fileMetadata: FileMetadata, - options?: {| previousVersion?: string, restoredFromVersionId?: string |}, + options?: SaveProjectOptions, actions: {| showAlert: ShowAlertFunction, showConfirmation: ShowConfirmFunction, diff --git a/newIDE/app/src/Utils/Serializer.js b/newIDE/app/src/Utils/Serializer.js index 8f5cc6b3b8b6..2ee60041d0c3 100644 --- a/newIDE/app/src/Utils/Serializer.js +++ b/newIDE/app/src/Utils/Serializer.js @@ -13,37 +13,16 @@ export function serializeToJSObject( serializable: gdSerializable, methodName: string = 'serializeTo' ) { - const startTime = Date.now(); const serializedElement = new gd.SerializerElement(); serializable[methodName](serializedElement); - const serializeToTime = Date.now(); - console.log( - '[Main thread] serializeTo done in: ', - serializeToTime - startTime, - 'ms' - ); // JSON.parse + toJSON is 30% faster than gd.Serializer.toJSObject. - const toJSONStartTime = Date.now(); const json = gd.Serializer.toJSON(serializedElement); - const toJSONTime = Date.now(); - console.log( - '[Main thread] toJSON done in: ', - toJSONTime - toJSONStartTime, - 'ms' - ); try { const object = JSON.parse(json); - const parseTime = Date.now(); - console.log('[Main thread] parse done in: ', parseTime - toJSONTime, 'ms'); serializedElement.delete(); - console.log( - '[Main thread] serializeToJSObject done in: ', - Date.now() - startTime, - 'ms' - ); return object; } catch (error) { serializedElement.delete(); @@ -119,25 +98,12 @@ export function serializeToJSON( serializable: gdSerializable, methodName: string = 'serializeTo' ): string { - const startTime = Date.now(); const serializedElement = new gd.SerializerElement(); serializable[methodName](serializedElement); - const serializeToTime = Date.now(); - console.log( - '[Main thread] serializeTo done in: ', - serializeToTime - startTime, - 'ms' - ); // toJSON is 20% faster than gd.Serializer.toJSObject + JSON.stringify. const json = gd.Serializer.toJSON(serializedElement); serializedElement.delete(); - const toJSONTime = Date.now(); - console.log( - '[Main thread] toJSON done in: ', - toJSONTime - serializeToTime, - 'ms' - ); return json; } From 7fb8bd4ca1b6dbf432db3d391c7632d2b0d8a637 Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Tue, 23 Dec 2025 16:18:11 +0100 Subject: [PATCH 08/10] Add setting for background serialization --- .../src/MainFrame/Preferences/PreferencesContext.js | 4 ++++ .../src/MainFrame/Preferences/PreferencesDialog.js | 10 ++++++++++ .../src/MainFrame/Preferences/PreferencesProvider.js | 12 ++++++++++++ newIDE/app/src/MainFrame/index.js | 6 +++++- 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js b/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js index d4d7d0d85caa..2fa29f62ff68 100644 --- a/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js +++ b/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js @@ -231,6 +231,7 @@ export type PreferencesValues = {| aiState: {| aiRequestId: string | null |}, automaticallyUseCreditsForAiRequests: boolean, hasSeenInGameEditorWarning: boolean, + useBackgroundSerializerForSaving: boolean, |}; /** @@ -343,6 +344,7 @@ export type Preferences = {| |}) => void, setAutomaticallyUseCreditsForAiRequests: (enabled: boolean) => void, setHasSeenInGameEditorWarning: (enabled: boolean) => void, + setUseBackgroundSerializerForSaving: (enabled: boolean) => void, |}; export const initialPreferences = { @@ -405,6 +407,7 @@ export const initialPreferences = { aiState: { aiRequestId: null }, automaticallyUseCreditsForAiRequests: false, hasSeenInGameEditorWarning: false, + useBackgroundSerializerForSaving: false, }, setLanguage: () => {}, setThemeName: () => {}, @@ -485,6 +488,7 @@ export const initialPreferences = { setAiState: ({ aiRequestId }: {| aiRequestId: string | null |}) => {}, setAutomaticallyUseCreditsForAiRequests: (enabled: boolean) => {}, setHasSeenInGameEditorWarning: (enabled: boolean) => {}, + setUseBackgroundSerializerForSaving: (enabled: boolean) => {}, }; const PreferencesContext = React.createContext(initialPreferences); diff --git a/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js b/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js index 8e4b454c8734..1cdadd2d3558 100644 --- a/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js +++ b/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js @@ -87,6 +87,7 @@ const PreferencesDialog = ({ setAutomaticallyUseCreditsForAiRequests, setShowCreateSectionByDefault, setHasSeenInGameEditorWarning, + setUseBackgroundSerializerForSaving, } = React.useContext(PreferencesContext); const initialUse3DEditor = React.useRef(values.use3DEditor); @@ -590,6 +591,15 @@ const PreferencesDialog = ({ t`Show a warning on deprecated actions and conditions` )} /> + {!!electron && ( { setHasSeenInGameEditorWarning: this._setHasSeenInGameEditorWarning.bind( this ), + setUseBackgroundSerializerForSaving: this._setUseBackgroundSerializerForSaving.bind( + this + ), }; componentDidMount() { @@ -1010,6 +1013,15 @@ export default class PreferencesProvider extends React.Component { ); } + _setUseBackgroundSerializerForSaving(newValue: boolean) { + this.setState( + state => ({ + values: { ...state.values, useBackgroundSerializerForSaving: newValue }, + }), + () => this._persistValuesToLocalStorage(this.state) + ); + } + _getEditorStateForProject(projectId: string) { return this.state.values.editorStateByProject[projectId]; } diff --git a/newIDE/app/src/MainFrame/index.js b/newIDE/app/src/MainFrame/index.js index 1f143128d966..0e6be03e1721 100644 --- a/newIDE/app/src/MainFrame/index.js +++ b/newIDE/app/src/MainFrame/index.js @@ -3906,6 +3906,7 @@ const MainFrame = (props: Props) => { ] ); + const saveWithBackgroundSerializer = preferences.values.useBackgroundSerializerForSaving; const saveProject = React.useCallback( async () => { if (!currentProject) return; @@ -3958,7 +3959,9 @@ const MainFrame = (props: Props) => { // store their values in variables now. const storageProviderInternalName = getStorageProvider().internalName; - const saveOptions: SaveProjectOptions = {}; + const saveOptions: SaveProjectOptions = { + useBackgroundSerializer: saveWithBackgroundSerializer, + }; if (cloudProjectRecoveryOpenedVersionId) { saveOptions.previousVersion = cloudProjectRecoveryOpenedVersionId; } else { @@ -4043,6 +4046,7 @@ const MainFrame = (props: Props) => { } }, [ + saveWithBackgroundSerializer, isSavingProject, currentProject, currentProjectRef, From 470df7cd9709bba4a5dd08cc5df033f0eab6eac7 Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Tue, 23 Dec 2025 16:25:16 +0100 Subject: [PATCH 09/10] Fix formatting --- newIDE/app/src/MainFrame/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/newIDE/app/src/MainFrame/index.js b/newIDE/app/src/MainFrame/index.js index 0e6be03e1721..bf5e7601d0f0 100644 --- a/newIDE/app/src/MainFrame/index.js +++ b/newIDE/app/src/MainFrame/index.js @@ -3906,7 +3906,8 @@ const MainFrame = (props: Props) => { ] ); - const saveWithBackgroundSerializer = preferences.values.useBackgroundSerializerForSaving; + const saveWithBackgroundSerializer = + preferences.values.useBackgroundSerializerForSaving; const saveProject = React.useCallback( async () => { if (!currentProject) return; From e37127ac432c2cf8b33f18498076b2e5f4f7084d Mon Sep 17 00:00:00 2001 From: Florian Rival Date: Tue, 23 Dec 2025 16:39:20 +0100 Subject: [PATCH 10/10] Clear pending background serialization requests --- newIDE/app/src/Utils/BackgroundSerializer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/newIDE/app/src/Utils/BackgroundSerializer.js b/newIDE/app/src/Utils/BackgroundSerializer.js index 60afccff4f09..32083650b27a 100644 --- a/newIDE/app/src/Utils/BackgroundSerializer.js +++ b/newIDE/app/src/Utils/BackgroundSerializer.js @@ -42,6 +42,7 @@ const getOrCreateBackgroundSerializerWorker = (): Worker => { ); return; } + pendingRequests.delete(data.requestId); if (data.type === 'DONE') { pending.resolve(data.result);