diff --git a/Core/GDCore/Serialization/BinarySerializer.cpp b/Core/GDCore/Serialization/BinarySerializer.cpp new file mode 100644 index 000000000000..68ebffd49a9c --- /dev/null +++ b/Core/GDCore/Serialization/BinarySerializer.cpp @@ -0,0 +1,291 @@ +/* + * GDevelop Core + * Copyright 2008-2016 Florian Rival (Florian.Rival@gmail.com). All rights + * reserved. This project is released under the MIT License. + */ + +#include "GDCore/Serialization/BinarySerializer.h" +#include "GDCore/Tools/Log.h" +#include +#include + +namespace gd { + +size_t BinarySerializer::lastBinarySnapshotSize = 0; + +using NodeType = BinarySerializer::NodeType; + +void BinarySerializer::SerializeToBinaryBuffer(const SerializerElement& element, + std::vector& outBuffer) { + // Reserve approximate size (heuristic: 1KB minimum) + outBuffer.clear(); + outBuffer.reserve(1024); + + // Write magic header and version + Write(outBuffer, static_cast(0x47444253)); // "GDBS" magic + Write(outBuffer, static_cast(1)); // Version 1 + + // Serialize the element tree + SerializeElement(element, outBuffer); +} + +void BinarySerializer::SerializeElement(const SerializerElement& element, + std::vector& buffer) { + Write(buffer, NodeType::Element); + + // Serialize value + if (element.IsValueUndefined()) { + Write(buffer, NodeType::ValueUndefined); + } else { + SerializeValue(element.GetValue(), buffer); + } + + // Serialize attributes + const auto& attributes = element.GetAllAttributes(); + Write(buffer, static_cast(attributes.size())); + for (const auto& attr : attributes) { + SerializeString(attr.first, buffer); + SerializeValue(attr.second, buffer); + } + + // Serialize array flags + Write(buffer, element.ConsideredAsArray()); + SerializeString(element.ConsideredAsArrayOf(), buffer); + + // Serialize children + const auto& children = element.GetAllChildren(); + Write(buffer, static_cast(children.size())); + for (const auto& child : children) { + SerializeString(child.first, buffer); // Child name + SerializeElement(*child.second, buffer); // Child element (recursive) + } +} + +void BinarySerializer::SerializeValue(const SerializerValue& value, + std::vector& buffer) { + if (value.IsBoolean()) { + Write(buffer, NodeType::ValueBool); + Write(buffer, value.GetBool()); + } else if (value.IsInt()) { + Write(buffer, NodeType::ValueInt); + Write(buffer, value.GetInt()); + } else if (value.IsDouble()) { + Write(buffer, NodeType::ValueDouble); + Write(buffer, value.GetDouble()); + } else if (value.IsString()) { + Write(buffer, NodeType::ValueString); + SerializeString(value.GetString(), buffer); + } else { + // Shouldn't happen, but handle gracefully + Write(buffer, NodeType::ValueUndefined); + } +} + +void BinarySerializer::SerializeString(const gd::String& str, + std::vector& buffer) { + // Convert to UTF-8 + std::string utf8 = str.ToUTF8(); + + // Write length + data + Write(buffer, static_cast(utf8.size())); + buffer.insert(buffer.end(), utf8.begin(), utf8.end()); +} + +bool BinarySerializer::DeserializeFromBinaryBuffer(const uint8_t* buffer, + size_t bufferSize, + SerializerElement& outElement) { + const uint8_t* ptr = buffer; + const uint8_t* end = buffer + bufferSize; + + // Read and verify magic header + uint32_t magic; + if (!Read(ptr, end, magic) || magic != 0x47444253) { + gd::LogError("Failed to deserialize binary snapshot: invalid magic."); + return false; // Invalid magic + } + + // Read version + uint32_t version; + if (!Read(ptr, end, version) || version != 1) { + gd::LogError("Failed to deserialize binary snapshot: unsupported version."); + return false; // Unsupported version + } + + // Deserialize element tree + return DeserializeElement(ptr, end, outElement); +} + +bool BinarySerializer::DeserializeElement(const uint8_t*& ptr, + const uint8_t* end, + SerializerElement& element) { + NodeType nodeType; + if (!Read(ptr, end, nodeType) || nodeType != NodeType::Element) { + gd::LogError("Failed to deserialize binary snapshot: invalid node type."); + return false; + } + + // Deserialize value + NodeType valueType; + if (!Read(ptr, end, valueType)) { + gd::LogError("Failed to deserialize binary snapshot: invalid value type."); + return false; + } + + if (valueType != NodeType::ValueUndefined) { + SerializerValue value; + if (!DeserializeValue(ptr, end, value, valueType)) return false; + element.SetValue(value); + } + + // Deserialize attributes + uint32_t attrCount; + if (!Read(ptr, end, attrCount)) return false; + + for (uint32_t i = 0; i < attrCount; ++i) { + gd::String attrName; + if (!DeserializeString(ptr, end, attrName)) return false; + + SerializerValue attrValue; + NodeType attrValueType; + if (!Read(ptr, end, attrValueType)) return false; + if (!DeserializeValue(ptr, end, attrValue, attrValueType)) return false; + + // Set attribute based on type + if (attrValue.IsBoolean()) { + element.SetAttribute(attrName, attrValue.GetBool()); + } else if (attrValue.IsInt()) { + element.SetAttribute(attrName, attrValue.GetInt()); + } else if (attrValue.IsDouble()) { + element.SetAttribute(attrName, attrValue.GetDouble()); + } else if (attrValue.IsString()) { + element.SetAttribute(attrName, attrValue.GetString()); + } + } + + // Deserialize array flags + bool isArray; + if (!Read(ptr, end, isArray)) return false; + if (isArray) element.ConsiderAsArray(); + + gd::String arrayOf; + if (!DeserializeString(ptr, end, arrayOf)) return false; + if (!arrayOf.empty()) { + element.ConsiderAsArrayOf(arrayOf); + } + + // Deserialize children + uint32_t childCount; + if (!Read(ptr, end, childCount)) return false; + + for (uint32_t i = 0; i < childCount; ++i) { + gd::String childName; + if (!DeserializeString(ptr, end, childName)) return false; + + SerializerElement& child = element.AddChild(childName); + if (!DeserializeElement(ptr, end, child)) return false; + } + + return true; +} + +bool BinarySerializer::DeserializeValue(const uint8_t*& ptr, + const uint8_t* end, + SerializerValue& value, + NodeType valueType) { + switch (valueType) { + case NodeType::ValueBool: { + bool boolVal; + if (!Read(ptr, end, boolVal)) return false; + value.SetBool(boolVal); + break; + } + case NodeType::ValueInt: { + int intVal; + if (!Read(ptr, end, intVal)) return false; + value.SetInt(intVal); + break; + } + case NodeType::ValueDouble: { + double doubleVal; + if (!Read(ptr, end, doubleVal)) return false; + value.SetDouble(doubleVal); + break; + } + case NodeType::ValueString: { + gd::String strVal; + if (!DeserializeString(ptr, end, strVal)) return false; + value.SetString(strVal); + break; + } + case NodeType::ValueUndefined: + // Value remains undefined + break; + default: + return false; // Unknown value type + } + + return true; +} + +bool BinarySerializer::DeserializeString(const uint8_t*& ptr, + const uint8_t* end, + gd::String& str) { + uint32_t length; + if (!Read(ptr, end, length)) return false; + + if (ptr + length > end) return false; + + std::string utf8(reinterpret_cast(ptr), length); + ptr += length; + + str = gd::String::FromUTF8(utf8); + return true; +} + +uintptr_t BinarySerializer::CreateBinarySnapshot(const SerializerElement& element) { + std::vector buffer; + SerializeToBinaryBuffer(element, buffer); + + lastBinarySnapshotSize = buffer.size(); + + // Allocate memory in Emscripten heap + uint8_t* heapBuffer = (uint8_t*)malloc(buffer.size()); + if (!heapBuffer) { + lastBinarySnapshotSize = 0; + return 0; + } + + std::memcpy(heapBuffer, buffer.data(), buffer.size()); + return reinterpret_cast(heapBuffer); +} + +size_t BinarySerializer::GetLastBinarySnapshotSize() { + return lastBinarySnapshotSize; +} + +void BinarySerializer::FreeBinarySnapshot(uintptr_t bufferPtr) { + if (bufferPtr) { + free(reinterpret_cast(bufferPtr)); + } +} + +SerializerElement* BinarySerializer::DeserializeBinarySnapshot(uintptr_t bufferPtr, + size_t size) { + if (!bufferPtr || size == 0) { + gd::LogError("Failed to deserialize binary snapshot: invalid buffer pointer or size."); + return nullptr; + } + + const uint8_t* buffer = reinterpret_cast(bufferPtr); + SerializerElement* element = new SerializerElement(); + + if (!DeserializeFromBinaryBuffer(buffer, size, *element)) { + gd::LogError("Failed to deserialize binary snapshot."); + delete element; + return nullptr; + } + + return element; +} + +} // namespace gd diff --git a/Core/GDCore/Serialization/BinarySerializer.h b/Core/GDCore/Serialization/BinarySerializer.h new file mode 100644 index 000000000000..28295be0e1a5 --- /dev/null +++ b/Core/GDCore/Serialization/BinarySerializer.h @@ -0,0 +1,125 @@ +/* + * GDevelop Core + * Copyright 2008-present Florian Rival (Florian.Rival@gmail.com). All rights + * reserved. This project is released under the MIT License. + */ + +#pragma once + +#include +#include +#include + +#include "GDCore/Serialization/SerializerElement.h" + +namespace gd { + +/** + * \brief Fast binary serialization/deserialization for SerializerElement trees. + * + * This format is optimized for speed and compactness, not for human readability + * or long-term storage. For transferring data between "threads" (web workers). + */ +class GD_CORE_API BinarySerializer { + public: + /** + * \brief Serialize a SerializerElement tree to a binary buffer. + * + * \param element The root element to serialize + * \param outBuffer Output buffer that will contain the binary data + */ + static void SerializeToBinaryBuffer(const SerializerElement& element, + std::vector& outBuffer); + + /** + * \brief Deserialize a binary buffer back to a SerializerElement tree. + * + * \param buffer The binary data + * \param bufferSize Size of the binary data + * \param outElement The output element (will be populated) + * \return true if successful, false if corrupted data + */ + static bool DeserializeFromBinaryBuffer(const uint8_t* buffer, + size_t bufferSize, + SerializerElement& outElement); + + /** \name Helpers + */ + ///@{ + /** + * \brief Create a binary snapshot and return the size. + * The binary data is written to Emscripten's heap at the returned pointer. + * \return Pointer to binary data in Emscripten heap (use GetLastBinarySnapshotSize to get size) + */ + static uintptr_t CreateBinarySnapshot(const SerializerElement& element); + + /** + * \brief Get the size of the last created binary snapshot. + * Must be called immediately after CreateBinarySnapshot. + */ + static size_t GetLastBinarySnapshotSize(); + + /** + * \brief Free a binary snapshot. + */ + static void FreeBinarySnapshot(uintptr_t bufferPtr); + + /** + * \brief Deserialize binary snapshot back to SerializerElement. + * \return New SerializerElement pointer (caller owns it) + */ + static SerializerElement* DeserializeBinarySnapshot(uintptr_t bufferPtr, + size_t size); + ///@} + + enum class NodeType : uint8_t { + Element = 0x01, + ValueUndefined = 0x02, + ValueBool = 0x03, + ValueInt = 0x04, + ValueDouble = 0x05, + ValueString = 0x06 + }; + + private: + // Internal serialization + static void SerializeElement(const SerializerElement& element, + std::vector& buffer); + static void SerializeValue(const SerializerValue& value, + std::vector& buffer); + static void SerializeString(const gd::String& str, + std::vector& buffer); + + // Internal deserialization + static bool DeserializeElement(const uint8_t*& ptr, + const uint8_t* end, + SerializerElement& element); + static bool DeserializeValue(const uint8_t*& ptr, + const uint8_t* end, + SerializerValue& value, + NodeType valueType); + static bool DeserializeString(const uint8_t*& ptr, + const uint8_t* end, + gd::String& str); + + // Helper to write primitive types + template + static void Write(std::vector& buffer, const T& value) { + const uint8_t* bytes = reinterpret_cast(&value); + buffer.insert(buffer.end(), bytes, bytes + sizeof(T)); + } + + // Helper to read primitive types + template + static bool Read(const uint8_t*& ptr, const uint8_t* end, T& value) { + if (ptr + sizeof(T) > end) return false; + std::memcpy(&value, ptr, sizeof(T)); + ptr += sizeof(T); + return true; + } + + // Store the last binary size for retrieval + static size_t lastBinarySnapshotSize; +}; + +} // namespace gd diff --git a/GDevelop.js/Bindings/Bindings.idl b/GDevelop.js/Bindings/Bindings.idl index 5ddbe2d01686..1dda800ed39e 100644 --- a/GDevelop.js/Bindings/Bindings.idl +++ b/GDevelop.js/Bindings/Bindings.idl @@ -1603,6 +1603,21 @@ interface Serializer { [Value] SerializerElement STATIC_FromJSON([Const] DOMString json); }; +[NoDelete] +interface BinarySerializer { + // Create a binary snapshot, returns pointer to buffer in Emscripten heap + unsigned long STATIC_CreateBinarySnapshot([Ref] SerializerElement element); + + // Get the size of the last created snapshot + unsigned long STATIC_GetLastBinarySnapshotSize(); + + // Free the binary snapshot + void STATIC_FreeBinarySnapshot(unsigned long bufferPtr); + + // Deserialize from a pointer in Emscripten heap + SerializerElement STATIC_DeserializeBinarySnapshot(unsigned long bufferPtr, unsigned long size); +}; + interface ObjectAssetSerializer { void STATIC_SerializeTo([Ref] Project project, [Const, Ref] gdObject obj, [Const] DOMString objectFullName, [Ref] SerializerElement element, diff --git a/GDevelop.js/Bindings/Wrapper.cpp b/GDevelop.js/Bindings/Wrapper.cpp index f5f45d1324d0..63176d39e2b7 100644 --- a/GDevelop.js/Bindings/Wrapper.cpp +++ b/GDevelop.js/Bindings/Wrapper.cpp @@ -99,6 +99,7 @@ #include #include #include +#include #include #include #include @@ -855,6 +856,10 @@ typedef std::vector VectorPropertyDescriptorChoice #define STATIC_GetDefaultMeasurementUnitByName GetDefaultMeasurementUnitByName #define STATIC_HasDefaultMeasurementUnitNamed HasDefaultMeasurementUnitNamed #define STATIC_GetEdgeAnchorFromString GetEdgeAnchorFromString +#define STATIC_CreateBinarySnapshot CreateBinarySnapshot +#define STATIC_GetLastBinarySnapshotSize GetLastBinarySnapshotSize +#define STATIC_FreeBinarySnapshot FreeBinarySnapshot +#define STATIC_DeserializeBinarySnapshot DeserializeBinarySnapshot // We postfix some methods with "At" as Javascript does not support overloading #define GetLayoutAt GetLayout diff --git a/GDevelop.js/CMakeLists.txt b/GDevelop.js/CMakeLists.txt index 4ace410de61e..fb0841083021 100644 --- a/GDevelop.js/CMakeLists.txt +++ b/GDevelop.js/CMakeLists.txt @@ -91,7 +91,7 @@ target_link_libraries(GD "-s TOTAL_MEMORY=48MB") # Get some initial memory size target_link_libraries(GD "-s ALLOW_MEMORY_GROWTH=1") target_link_libraries(GD "-s NODEJS_CATCH_EXIT=0") # Don't print the entire GDCore code on error when running in node target_link_libraries(GD "-s ERROR_ON_UNDEFINED_SYMBOLS=0") -target_link_libraries(GD "-s \"EXPORTED_FUNCTIONS=['_free']\"") +target_link_libraries(GD "-s \"EXPORTED_FUNCTIONS=['_free','_malloc']\"") if("${GDEVELOPJS_BUILD_VARIANT}" STREQUAL "dev") # Disable optimizations at linking time for slightly faster builds. diff --git a/GDevelop.js/__tests__/Serializer.js b/GDevelop.js/__tests__/Serializer.js index 1a4495503476..d0a72e1b0f06 100644 --- a/GDevelop.js/__tests__/Serializer.js +++ b/GDevelop.js/__tests__/Serializer.js @@ -1,13 +1,13 @@ const initializeGDevelopJs = require('../../Binaries/embuild/GDevelop.js/libGD.js'); -describe('libGD.js object serialization', function() { +describe('libGD.js object serialization', function () { let gd = null; beforeAll(async () => { gd = await initializeGDevelopJs(); }); - describe('gd.SerializerElement', function() { - it('should support operations on its value', function() { + describe('gd.SerializerElement', function () { + it('should support operations on its value', function () { var element = new gd.SerializerElement(); element.setStringValue('aaa'); expect(element.getStringValue()).toBe('aaa'); @@ -18,7 +18,7 @@ describe('libGD.js object serialization', function() { element.setDoubleValue(123.457); expect(element.getDoubleValue()).toBeCloseTo(123.457); }); - it('should cast values from a type to another', function() { + it('should cast values from a type to another', function () { var element = new gd.SerializerElement(); element.setStringValue('123'); expect(element.getStringValue()).toBe('123'); @@ -30,7 +30,7 @@ describe('libGD.js object serialization', function() { element.setBoolValue(false); expect(element.getBoolValue()).toBe(false); }); - it('should support operations on its children', function() { + it('should support operations on its children', function () { var element = new gd.SerializerElement(); expect(element.hasChild('Missing')).toBe(false); @@ -44,16 +44,13 @@ describe('libGD.js object serialization', function() { element.setChild('Child2', child2); expect( - element - .getChild('Child2') - .getChild('subChild') - .getStringValue() + element.getChild('Child2').getChild('subChild').getStringValue() ).toBe('Hello world!'); }); }); - describe('gd.Serializer', function() { - it('should serialize a Text Object', function() { + describe('gd.Serializer', function () { + it('should serialize a Text Object', function () { var obj = new gd.TextObject('testObject'); obj.setText('Text of the object, with 官话 characters'); @@ -69,31 +66,25 @@ describe('libGD.js object serialization', function() { }); }); - describe('gd.Serializer.fromJSON and gd.Serializer.toJSON', function() { + describe('gd.Serializer.fromJSON and gd.Serializer.toJSON', function () { const checkJsonParseAndStringify = (json) => { const element = gd.Serializer.fromJSON(json); const outputJson = gd.Serializer.toJSON(element); expect(outputJson).toBe(json); - } + }; - it('should unserialize and reserialize JSON (string)', function() { - checkJsonParseAndStringify( - '"a"' - ); - checkJsonParseAndStringify( - '"String with 官话 characters"' - ); - checkJsonParseAndStringify( - '""' - ); + it('should unserialize and reserialize JSON (string)', function () { + checkJsonParseAndStringify('"a"'); + checkJsonParseAndStringify('"String with 官话 characters"'); + checkJsonParseAndStringify('""'); }); - it('should unserialize and reserialize JSON (objects)', function() { + it('should unserialize and reserialize JSON (objects)', function () { checkJsonParseAndStringify( '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3.0},"things":{"0":{"name":"layout0","referenceTo":"/layouts/layout"},"1":{"name":"layout1","referenceTo":"/layouts/layout"},"2":{"name":"layout2","referenceTo":"/layouts/layout"},"3":{"name":"layout3","referenceTo":"/layouts/layout"},"4":{"name":"layout4","referenceTo":"/layouts/layout"}}}' ); }); - it('should unserialize and reserialize JSON (arrays)', function() { + it('should unserialize and reserialize JSON (arrays)', function () { checkJsonParseAndStringify('[]'); checkJsonParseAndStringify('[1]'); checkJsonParseAndStringify('[1,2]'); @@ -106,32 +97,26 @@ describe('libGD.js object serialization', function() { // TODO: test failures - describe('gd.Serializer.fromJSObject and gd.Serializer.toJSObject', function() { + describe('gd.Serializer.fromJSObject and gd.Serializer.toJSObject', function () { const checkJsonParseAndStringify = (json) => { const object = JSON.parse(json); const element = gd.Serializer.fromJSObject(object); const outputObject = gd.Serializer.toJSObject(element); expect(JSON.stringify(outputObject)).toBe(json); - } + }; - it('should unserialize and reserialize JSON (string)', function() { - checkJsonParseAndStringify( - '"a"' - ); - checkJsonParseAndStringify( - '"String with 官话 characters"' - ); - checkJsonParseAndStringify( - '""' - ); + it('should unserialize and reserialize JSON (string)', function () { + checkJsonParseAndStringify('"a"'); + checkJsonParseAndStringify('"String with 官话 characters"'); + checkJsonParseAndStringify('""'); }); - it('should unserialize and reserialize JSON (objects)', function() { + it('should unserialize and reserialize JSON (objects)', function () { checkJsonParseAndStringify( '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3},"things":{"0":{"name":"layout0","referenceTo":"/layouts/layout"},"1":{"name":"layout1","referenceTo":"/layouts/layout"},"2":{"name":"layout2","referenceTo":"/layouts/layout"},"3":{"name":"layout3","referenceTo":"/layouts/layout"},"4":{"name":"layout4","referenceTo":"/layouts/layout"}}}' ); }); - it('should unserialize and reserialize JSON (arrays)', function() { + it('should unserialize and reserialize JSON (arrays)', function () { checkJsonParseAndStringify('[]'); checkJsonParseAndStringify('[1]'); checkJsonParseAndStringify('[1,2]'); @@ -141,4 +126,150 @@ describe('libGD.js object serialization', function() { checkJsonParseAndStringify('{"7":[],"a":[1,2,{"b":3},{"c":[4,5]},6]}'); }); }); + + describe('gd.BinarySerializer', function () { + const serializeToBinarySnapshot = (serializerElement) => { + // Create binary snapshot + const binaryPtr = + gd.BinarySerializer.createBinarySnapshot(serializerElement); + const binarySize = gd.BinarySerializer.getLastBinarySnapshotSize(); + + if (!binaryPtr) { + throw new Error('Failed to create binary snapshot.'); + } + + const binaryView = new Uint8Array( + gd.HEAPU8.buffer, + binaryPtr, + binarySize + ); + // Copy the buffer out of the WASM heap to simulate it was transferred. + const binaryBuffer = binaryView.slice(); + + gd.BinarySerializer.freeBinarySnapshot(binaryPtr); + + return binaryBuffer; + }; + + const serializeJsonToBinarySnapshot = (json) => { + const element = gd.Serializer.fromJSON(json); + // Do NOT delete element, it's a static value. + + return serializeToBinarySnapshot(element); + }; + + const unserializeBinarySnapshotToJson = (binaryBuffer) => { + const binaryArray = + binaryBuffer instanceof Uint8Array + ? binaryBuffer + : new Uint8Array(binaryBuffer); + const binarySize = binaryArray.byteLength || binaryArray.length; + + // Allocate memory in Emscripten heap and copy binary data + const binaryPtr = gd._malloc(binarySize); + gd.HEAPU8.set(binaryArray, binaryPtr); + + const element = gd.BinarySerializer.deserializeBinarySnapshot( + binaryPtr, + binarySize + ); + + // Free the input buffer + gd._free(binaryPtr); + + if (element.ptr === 0) { + throw new Error('Failed to deserialize binary snapshot.'); + } + + const json = gd.Serializer.toJSON(element); + element.delete(); + return json; + }; + + const checkBinaryRoundTrip = (json) => { + const binaryBuffer = serializeJsonToBinarySnapshot(json); + expect(binaryBuffer).toBeInstanceOf(Uint8Array); + expect(binaryBuffer.length).toBeGreaterThan(0); + + const outputJson = unserializeBinarySnapshotToJson(binaryBuffer); + expect(outputJson).toBe(json); + }; + + it('should round-trip simple values', function () { + checkBinaryRoundTrip('"hello"'); + checkBinaryRoundTrip('"hello"'); + checkBinaryRoundTrip('123'); + checkBinaryRoundTrip('123.456'); + checkBinaryRoundTrip('true'); + checkBinaryRoundTrip('false'); + }); + + it('should round-trip strings with unicode characters', function () { + checkBinaryRoundTrip('"String with 官话 characters"'); + checkBinaryRoundTrip('"Émojis: 🎮🎲🎯"'); + }); + + it('should round-trip objects', function () { + checkBinaryRoundTrip('{}'); + checkBinaryRoundTrip('{"a":"b"}'); + checkBinaryRoundTrip('{"a":{"nested":"value"}}'); + checkBinaryRoundTrip( + '{"a":{"a1":{"name":"","referenceTo":"/a/a1"}},"b":{"b1":"world"},"c":{"c1":3.0}}' + ); + }); + + it('should round-trip arrays', function () { + checkBinaryRoundTrip('[]'); + checkBinaryRoundTrip('[1]'); + checkBinaryRoundTrip('[1,2,3]'); + checkBinaryRoundTrip('[{"a":1},{"b":2}]'); + checkBinaryRoundTrip('{"items":[1,2,3],"nested":[{"x":1},{"y":2}]}'); + }); + + it('should round-trip a complex object like a Text Object', function () { + const obj = new gd.TextObject('testObject'); + obj.setText('Text with 官话 characters'); + + const serializedElement = new gd.SerializerElement(); + obj.serializeTo(serializedElement); + + const binaryBuffer = serializeToBinarySnapshot(serializedElement); + const jsonFromBinaryBuffer = + unserializeBinarySnapshotToJson(binaryBuffer); + + // Compare JSON output from the original SerializerElement + // with the JSON obtained from the binary buffer: + const originalJson = gd.Serializer.toJSON(serializedElement); + serializedElement.delete(); + obj.delete(); + + expect(jsonFromBinaryBuffer).toBe(originalJson); + }); + + it('should round-trip a complex object like a Project', function () { + const project = new gd.ProjectHelper.createNewGDJSProject(); + const layout = project.insertNewLayout('Scene', 0); + layout.getObjects().insertNewObject(project, 'Sprite', 'Object1', 0); + layout.getObjects().insertNewObject(project, 'Sprite', 'Object2', 1); + const instance1 = layout.getInitialInstances().insertNewInitialInstance(); + const instance2 = layout.getInitialInstances().insertNewInitialInstance(); + instance1.setObjectName('Object1'); + instance2.setObjectName('Object2'); + + const serializedElement = new gd.SerializerElement(); + project.serializeTo(serializedElement); + + const binaryBuffer = serializeToBinarySnapshot(serializedElement); + const jsonFromBinaryBuffer = + unserializeBinarySnapshotToJson(binaryBuffer); + + // Compare JSON output from the original SerializerElement + // with the JSON obtained from the binary buffer: + const originalJson = gd.Serializer.toJSON(serializedElement); + serializedElement.delete(); + project.delete(); + + expect(jsonFromBinaryBuffer).toBe(originalJson); + }); + }); }); diff --git a/GDevelop.js/scripts/generate-dts.mjs b/GDevelop.js/scripts/generate-dts.mjs index ff036247d226..b04d9fbe4efd 100644 --- a/GDevelop.js/scripts/generate-dts.mjs +++ b/GDevelop.js/scripts/generate-dts.mjs @@ -453,6 +453,10 @@ export function compare(object1: T, object2: T): boo */ export function destroy(object: EmscriptenObject): void; +export function _malloc(size: number): number; +export function _free(ptr: number): void; +export const HEAPU8: Uint8Array; + export as namespace gd; declare global { diff --git a/GDevelop.js/scripts/generate-types.js b/GDevelop.js/scripts/generate-types.js index 3705f1e9357b..715b5fbb5549 100644 --- a/GDevelop.js/scripts/generate-types.js +++ b/GDevelop.js/scripts/generate-types.js @@ -288,8 +288,13 @@ type CustomObjectConfiguration_EdgeAnchor = 0 | 1 | 2 | 3 | 4` 'declare class libGDevelop {', ' getPointer(gdEmscriptenObject): number;', ' castObject(gdEmscriptenObject, Class): T;', + ' wrapPointer(ptr: number, objectClass: Class): T;', ' compare(gdEmscriptenObject, gdEmscriptenObject): boolean;', '', + ' _malloc(size: number): number;', + ' _free(ptr: number): void;', + ' HEAPU8: Uint8Array;', + '', ' getTypeOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string;', ' getTypeOfBehavior(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string;', ' getBehaviorsOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): gdVectorString;', diff --git a/GDevelop.js/types.d.ts b/GDevelop.js/types.d.ts index 4f40c764e896..dcb868473aa6 100644 --- a/GDevelop.js/types.d.ts +++ b/GDevelop.js/types.d.ts @@ -1325,6 +1325,13 @@ export class Serializer extends EmscriptenObject { static toJSObject(element: gdSerializerElement): any; } +export class BinarySerializer extends EmscriptenObject { + static createBinarySnapshot(element: SerializerElement): number; + static getLastBinarySnapshotSize(): number; + static freeBinarySnapshot(bufferPtr: number): void; + static deserializeBinarySnapshot(bufferPtr: number, size: number): SerializerElement; +} + export class ObjectAssetSerializer extends EmscriptenObject { static serializeTo(project: Project, obj: gdObject, objectFullName: string, element: SerializerElement, usedResourceNames: VectorString): void; } @@ -3228,6 +3235,10 @@ export function compare(object1: T, object2: T): boo */ export function destroy(object: EmscriptenObject): void; +export function _malloc(size: number): number; +export function _free(ptr: number): void; +export const HEAPU8: Uint8Array; + export as namespace gd; declare global { diff --git a/GDevelop.js/types/gdbinaryserializer.js b/GDevelop.js/types/gdbinaryserializer.js new file mode 100644 index 000000000000..17932d328b73 --- /dev/null +++ b/GDevelop.js/types/gdbinaryserializer.js @@ -0,0 +1,9 @@ +// Automatically generated by GDevelop.js/scripts/generate-types.js +declare class gdBinarySerializer { + static createBinarySnapshot(element: gdSerializerElement): number; + static getLastBinarySnapshotSize(): number; + static freeBinarySnapshot(bufferPtr: number): void; + static deserializeBinarySnapshot(bufferPtr: number, size: number): gdSerializerElement; + delete(): void; + ptr: number; +}; \ No newline at end of file diff --git a/GDevelop.js/types/libgdevelop.js b/GDevelop.js/types/libgdevelop.js index ffc1b62b68fe..eab9afa3582f 100644 --- a/GDevelop.js/types/libgdevelop.js +++ b/GDevelop.js/types/libgdevelop.js @@ -2,8 +2,13 @@ declare class libGDevelop { getPointer(gdEmscriptenObject): number; castObject(gdEmscriptenObject, Class): T; + wrapPointer(ptr: number, objectClass: Class): T; compare(gdEmscriptenObject, gdEmscriptenObject): boolean; + _malloc(size: number): number; + _free(ptr: number): void; + HEAPU8: Uint8Array; + getTypeOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string; getTypeOfBehavior(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): string; getBehaviorsOfObject(globalObjectsContainer: gdObjectsContainer, objectsContainer: gdObjectsContainer, objectName: string, searchInGroups: boolean): gdVectorString; @@ -141,6 +146,7 @@ declare class libGDevelop { SerializerElement: Class; SharedPtrSerializerElement: Class; Serializer: Class; + BinarySerializer: Class; ObjectAssetSerializer: Class; InstructionsList: Class; Instruction: Class; diff --git a/newIDE/app/scripts/make-service-worker.js b/newIDE/app/scripts/make-service-worker.js index 33569ded1884..11021c16d729 100644 --- a/newIDE/app/scripts/make-service-worker.js +++ b/newIDE/app/scripts/make-service-worker.js @@ -56,6 +56,7 @@ const buildSW = () => { 'static/js/locales-*.js', // Locales. 'static/js/!local-app*.js', // Exclude electron app. 'static/js/Resource3DPreview.worker.*.js', // Include the 3D preview worker + 'static/js/BackgroundSerializer.worker.*.js', // Include the serializer worker // ...But not libGD.js/wasm (there are cached with their URL // query string that depends on the VersionMetadata, see below). diff --git a/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js b/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js index d4d7d0d85caa..2fa29f62ff68 100644 --- a/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js +++ b/newIDE/app/src/MainFrame/Preferences/PreferencesContext.js @@ -231,6 +231,7 @@ export type PreferencesValues = {| aiState: {| aiRequestId: string | null |}, automaticallyUseCreditsForAiRequests: boolean, hasSeenInGameEditorWarning: boolean, + useBackgroundSerializerForSaving: boolean, |}; /** @@ -343,6 +344,7 @@ export type Preferences = {| |}) => void, setAutomaticallyUseCreditsForAiRequests: (enabled: boolean) => void, setHasSeenInGameEditorWarning: (enabled: boolean) => void, + setUseBackgroundSerializerForSaving: (enabled: boolean) => void, |}; export const initialPreferences = { @@ -405,6 +407,7 @@ export const initialPreferences = { aiState: { aiRequestId: null }, automaticallyUseCreditsForAiRequests: false, hasSeenInGameEditorWarning: false, + useBackgroundSerializerForSaving: false, }, setLanguage: () => {}, setThemeName: () => {}, @@ -485,6 +488,7 @@ export const initialPreferences = { setAiState: ({ aiRequestId }: {| aiRequestId: string | null |}) => {}, setAutomaticallyUseCreditsForAiRequests: (enabled: boolean) => {}, setHasSeenInGameEditorWarning: (enabled: boolean) => {}, + setUseBackgroundSerializerForSaving: (enabled: boolean) => {}, }; const PreferencesContext = React.createContext(initialPreferences); diff --git a/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js b/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js index 8e4b454c8734..1cdadd2d3558 100644 --- a/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js +++ b/newIDE/app/src/MainFrame/Preferences/PreferencesDialog.js @@ -87,6 +87,7 @@ const PreferencesDialog = ({ setAutomaticallyUseCreditsForAiRequests, setShowCreateSectionByDefault, setHasSeenInGameEditorWarning, + setUseBackgroundSerializerForSaving, } = React.useContext(PreferencesContext); const initialUse3DEditor = React.useRef(values.use3DEditor); @@ -590,6 +591,15 @@ const PreferencesDialog = ({ t`Show a warning on deprecated actions and conditions` )} /> + {!!electron && ( { setHasSeenInGameEditorWarning: this._setHasSeenInGameEditorWarning.bind( this ), + setUseBackgroundSerializerForSaving: this._setUseBackgroundSerializerForSaving.bind( + this + ), }; componentDidMount() { @@ -1010,6 +1013,15 @@ export default class PreferencesProvider extends React.Component { ); } + _setUseBackgroundSerializerForSaving(newValue: boolean) { + this.setState( + state => ({ + values: { ...state.values, useBackgroundSerializerForSaving: newValue }, + }), + () => this._persistValuesToLocalStorage(this.state) + ); + } + _getEditorStateForProject(projectId: string) { return this.state.values.editorStateByProject[projectId]; } diff --git a/newIDE/app/src/MainFrame/index.js b/newIDE/app/src/MainFrame/index.js index 48ed7957612a..bf5e7601d0f0 100644 --- a/newIDE/app/src/MainFrame/index.js +++ b/newIDE/app/src/MainFrame/index.js @@ -109,6 +109,7 @@ import { type SaveAsOptions, type FileMetadataAndStorageProviderName, type ResourcesActionsMenuBuilder, + type SaveProjectOptions, } from '../ProjectsStorage'; import OpenFromStorageProviderDialog from '../ProjectsStorage/OpenFromStorageProviderDialog'; import SaveToStorageProviderDialog from '../ProjectsStorage/SaveToStorageProviderDialog'; @@ -3905,6 +3906,8 @@ const MainFrame = (props: Props) => { ] ); + const saveWithBackgroundSerializer = + preferences.values.useBackgroundSerializerForSaving; const saveProject = React.useCallback( async () => { if (!currentProject) return; @@ -3924,10 +3927,7 @@ const MainFrame = (props: Props) => { } const storageProviderOperations = getStorageProviderOperations(); - const { - onSaveProject, - canFileMetadataBeSafelySaved, - } = storageProviderOperations; + const { onSaveProject } = storageProviderOperations; if (!onSaveProject) { return saveProjectAs(); } @@ -3947,15 +3947,6 @@ const MainFrame = (props: Props) => { message: t`You're trying to save changes made to a previous version of your project. If you continue, it will be used as the new latest version.`, }); if (!shouldRestoreCheckedOutVersion) return; - } else if (canFileMetadataBeSafelySaved) { - const canProjectBeSafelySaved = await canFileMetadataBeSafelySaved( - currentFileMetadata, - { - showAlert, - showConfirmation, - } - ); - if (!canProjectBeSafelySaved) return; } _showSnackMessage(i18n._(t`Saving...`), null); @@ -3969,9 +3960,13 @@ const MainFrame = (props: Props) => { // store their values in variables now. const storageProviderInternalName = getStorageProvider().internalName; - const saveOptions = {}; + const saveOptions: SaveProjectOptions = { + useBackgroundSerializer: saveWithBackgroundSerializer, + }; if (cloudProjectRecoveryOpenedVersionId) { saveOptions.previousVersion = cloudProjectRecoveryOpenedVersionId; + } else { + saveOptions.previousVersion = currentFileMetadata.version; } if (checkedOutVersionStatus) { saveOptions.restoredFromVersionId = @@ -3980,7 +3975,11 @@ const MainFrame = (props: Props) => { const { wasSaved, fileMetadata } = await onSaveProject( currentProject, currentFileMetadata, - saveOptions + saveOptions, + { + showAlert, + showConfirmation, + } ); if (wasSaved) { @@ -4048,6 +4047,7 @@ const MainFrame = (props: Props) => { } }, [ + saveWithBackgroundSerializer, isSavingProject, currentProject, currentProjectRef, diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js index 9e99a81deddb..e1a99511b9b7 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudProjectWriter.js @@ -1,17 +1,24 @@ // @flow import * as React from 'react'; import { type AuthenticatedUser } from '../../Profile/AuthenticatedUserContext'; -import { type FileMetadata, type SaveAsLocation, type SaveAsOptions } from '..'; +import { + type FileMetadata, + type SaveAsLocation, + type SaveAsOptions, + type SaveProjectOptions, +} from '..'; import { CLOUD_PROJECT_NAME_MAX_LENGTH, commitVersion, createCloudProject, getCredentialsForCloudProject, + getPresignedUrlForVersionUpload, updateCloudProject, } from '../../Utils/GDevelopServices/Project'; import type { $AxiosError } from 'axios'; import type { MessageDescriptor } from '../../Utils/i18n/MessageDescriptor.flow'; import { serializeToJSON } from '../../Utils/Serializer'; +import { serializeToJSONInBackground } from '../../Utils/BackgroundSerializer'; import { t } from '@lingui/macro'; import { createZipWithSingleTextFile, @@ -22,14 +29,47 @@ import { getProjectCache } from './CloudProjectOpener'; import { retryIfFailed } from '../../Utils/RetryIfFailed'; import { extractGDevelopApiErrorStatusAndCode } from '../../Utils/GDevelopServices/Errors'; import SaveAsOptionsDialog from '../SaveAsOptionsDialog'; +import { + type ShowAlertFunction, + type ShowConfirmFunction, +} from '../../UI/Alert/AlertContext'; +import { + getCloudProject, + type CloudProjectWithUserAccessInfo, +} from '../../Utils/GDevelopServices/Project'; +import { format } from 'date-fns'; +import { getUserPublicProfile } from '../../Utils/GDevelopServices/User'; + +const zipProject = async ({ + project, + useBackgroundSerializer, +}: { + project: gdProject, + useBackgroundSerializer: boolean, +}): Promise<{ zippedProject: Blob, projectJson: string }> => { + const startTime = Date.now(); + + let projectJson: string; + if (useBackgroundSerializer) { + projectJson = await serializeToJSONInBackground(project); + } else { + projectJson = serializeToJSON(project); + } + + const serializeToJSONEndTime = Date.now(); -const zipProject = async (project: gdProject): Promise<[Blob, string]> => { - const projectJson = serializeToJSON(project); const zippedProject = await createZipWithSingleTextFile( projectJson, 'game.json' ); - return [zippedProject, projectJson]; + + console.log( + `[CloudProjectWriter] Zipping done in ${Date.now() - + startTime}ms (including ${serializeToJSONEndTime - startTime}ms for ${ + useBackgroundSerializer ? 'background' : 'main' + } thread serialization).` + ); + return { zippedProject, projectJson }; }; const checkZipContent = async ( @@ -47,7 +87,7 @@ const checkZipContent = async ( } }; -const zipProjectAndCommitVersion = async ({ +const zipAndPrepareProjectVersionForCommit = async ({ authenticatedUser, project, cloudProjectId, @@ -56,19 +96,51 @@ const zipProjectAndCommitVersion = async ({ authenticatedUser: AuthenticatedUser, project: gdProject, cloudProjectId: string, - options?: {| previousVersion?: string, restoredFromVersionId?: string |}, -|}): Promise => { - const [zippedProject, projectJson] = await zipProject(project); + options?: SaveProjectOptions, +|}): Promise<{| + presignedUrl: string, + zippedProject: Blob, +|}> => { + const [presignedUrl, { zippedProject, projectJson }] = await Promise.all([ + getPresignedUrlForVersionUpload(authenticatedUser, cloudProjectId), + zipProject({ + project, + useBackgroundSerializer: !!options && !!options.useBackgroundSerializer, + }), + ]); + const archiveIsSane = await checkZipContent(zippedProject, projectJson); if (!archiveIsSane) { throw new Error('Project compression failed before saving the project.'); } + if (!presignedUrl) { + throw new Error( + 'No presigned url was returned from getting presigned url for version upload api call.' + ); + } + + return { presignedUrl, zippedProject }; +}; +const commitProjectVersion = async ({ + authenticatedUser, + presignedUrl, + zippedProject, + cloudProjectId, + options, +}: {| + authenticatedUser: AuthenticatedUser, + presignedUrl: string, + zippedProject: Blob, + cloudProjectId: string, + options?: SaveProjectOptions, +|}): Promise => { const newVersion = await retryIfFailed({ times: 2 }, () => commitVersion({ authenticatedUser, cloudProjectId, zippedProject, + presignedUrl, previousVersion: options ? options.previousVersion : null, restoredFromVersionId: options ? options.restoredFromVersionId : null, }) @@ -81,13 +153,18 @@ export const generateOnSaveProject = ( ) => async ( project: gdProject, fileMetadata: FileMetadata, - options?: {| previousVersion?: string, restoredFromVersionId?: string |} + options?: SaveProjectOptions, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} ) => { const cloudProjectId = fileMetadata.fileIdentifier; const gameId = project.getProjectUuid(); const now = Date.now(); if (!fileMetadata.gameId) { + // This is a rare case so we don't do it in parallel with the rest. console.info('Game id was never set, updating the cloud project.'); try { await updateCloudProject(authenticatedUser, cloudProjectId, { @@ -98,9 +175,30 @@ export const generateOnSaveProject = ( // Do not throw, as this is not a blocking error. } } - const newVersion = await zipProjectAndCommitVersion({ + + // Do as much as possible in parallel: + const [canBeSafelySaved, { presignedUrl, zippedProject }] = await Promise.all( + [ + // Check (with a network call) if the project can be safely saved (not modified by someone else). + canFileMetadataBeSafelySaved(authenticatedUser, fileMetadata, actions), + // At the same time, serialize & zip the project and also get (with a network call) a presigned url to upload the project version. + zipAndPrepareProjectVersionForCommit({ + authenticatedUser, + project, + cloudProjectId, + options, + }), + ] + ); + if (!canBeSafelySaved) { + // Abort saving (nothing was "committed" or persisted, the signed urls were not used). + return { wasSaved: false, fileMetadata: fileMetadata }; + } + + const newVersion = await commitProjectVersion({ authenticatedUser, - project, + presignedUrl, + zippedProject, cloudProjectId, options, }); @@ -136,14 +234,22 @@ export const generateOnChangeProjectProperty = ( ): Promise => { if (!authenticatedUser.authenticated) return null; try { - await updateCloudProject( - authenticatedUser, - fileMetadata.fileIdentifier, - properties - ); - const newVersion = await zipProjectAndCommitVersion({ + const [, { presignedUrl, zippedProject }] = await Promise.all([ + updateCloudProject( + authenticatedUser, + fileMetadata.fileIdentifier, + properties + ), + zipAndPrepareProjectVersionForCommit({ + authenticatedUser, + project, + cloudProjectId: fileMetadata.fileIdentifier, + }), + ]); + const newVersion = await commitProjectVersion({ authenticatedUser, - project, + presignedUrl, + zippedProject, cloudProjectId: fileMetadata.fileIdentifier, }); if (!newVersion) { @@ -277,11 +383,20 @@ export const generateOnSaveProjectAs = ( // Commit the changes to the newly created cloud project. await getCredentialsForCloudProject(authenticatedUser, cloudProjectId); - const newVersion = await zipProjectAndCommitVersion({ + const { + presignedUrl, + zippedProject, + } = await zipAndPrepareProjectVersionForCommit({ authenticatedUser, project, cloudProjectId, }); + const newVersion = await commitProjectVersion({ + authenticatedUser, + presignedUrl, + zippedProject, + cloudProjectId, + }); if (!newVersion) throw new Error('No version id was returned from committing api call.'); @@ -355,3 +470,62 @@ export const generateOnAutoSaveProject = ( ); } : undefined; + +const canFileMetadataBeSafelySaved = async ( + authenticatedUser: AuthenticatedUser, + fileMetadata: FileMetadata, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} +) => { + // If the project is saved on the cloud, first fetch it. + // If the version of the project opened is different than the last version of the cloud project, + // it means that the project was modified by someone else. In this case, we should warn + // the user and ask them if they want to overwrite the changes. + const cloudProjectId = fileMetadata.fileIdentifier; + const openedProjectVersion = fileMetadata.version; + const cloudProject: ?CloudProjectWithUserAccessInfo = await getCloudProject( + authenticatedUser, + cloudProjectId + ); + if (!cloudProject) { + await actions.showAlert({ + title: t`Unable to save the project`, + message: t`The project could not be saved. Please try again later.`, + }); + return false; + } + const { currentVersion, committedAt } = cloudProject; + if ( + openedProjectVersion && + currentVersion && // should always be defined. + committedAt && // should always be defined. + currentVersion !== openedProjectVersion + ) { + let lastUsernameWhoModifiedProject = null; + const committedAtDate = new Date(committedAt); + const formattedDate = format(committedAtDate, 'dd-MM-yyyy'); + const formattedTime = format(committedAtDate, 'HH:mm:ss'); + const lastCommittedBy = cloudProject.lastCommittedBy; + if (lastCommittedBy) { + const lastUser = await getUserPublicProfile(lastCommittedBy); + if (lastUser) { + lastUsernameWhoModifiedProject = lastUser.username; + } + } + const answer = await actions.showConfirmation({ + title: t`Project was modified`, + message: lastUsernameWhoModifiedProject + ? t`This project was modified by ${lastUsernameWhoModifiedProject} on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?` + : t`This project was modified by someone else on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?`, + level: 'warning', + confirmButtonLabel: t`Overwrite`, + makeDismissButtonPrimary: true, + }); + + return answer; + } + + return true; +}; diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudStoageProviderInternalName.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudStoageProviderInternalName.js new file mode 100644 index 000000000000..fde3f9742471 --- /dev/null +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/CloudStoageProviderInternalName.js @@ -0,0 +1,3 @@ +// @flow + +export const cloudStorageProviderInternalName = 'Cloud'; diff --git a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js index 4e31ec0155da..e6cdbb7b5c1f 100644 --- a/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js +++ b/newIDE/app/src/ProjectsStorage/CloudStorageProvider/index.js @@ -1,7 +1,7 @@ // @flow import * as React from 'react'; import { t } from '@lingui/macro'; -import { type StorageProvider, type FileMetadata } from '../index'; +import { type StorageProvider } from '../index'; import { generateOnChangeProjectProperty, generateOnSaveProject, @@ -25,16 +25,7 @@ import { } from './CloudProjectOpener'; import Cloud from '../../UI/CustomSvgIcons/Cloud'; import { generateGetResourceActions } from './CloudProjectResourcesHandler'; -import { - type ShowAlertFunction, - type ShowConfirmFunction, -} from '../../UI/Alert/AlertContext'; -import { - getCloudProject, - type CloudProjectWithUserAccessInfo, -} from '../../Utils/GDevelopServices/Project'; -import { format } from 'date-fns'; -import { getUserPublicProfile } from '../../Utils/GDevelopServices/User'; +import { cloudStorageProviderInternalName } from './CloudStoageProviderInternalName'; const isURL = (filename: string) => { return ( @@ -47,7 +38,7 @@ const isURL = (filename: string) => { }; export default ({ - internalName: 'Cloud', + internalName: cloudStorageProviderInternalName, name: t`GDevelop Cloud`, renderIcon: props => , hiddenInOpenDialog: true, @@ -89,63 +80,6 @@ export default ({ return t`An error occurred when opening the project. Check that your internet connection is working and that your browser allows the use of cookies.`; }, getWriteErrorMessage, - canFileMetadataBeSafelySaved: async ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => { - // If the project is saved on the cloud, first fetch it. - // If the version of the project opened is different than the last version of the cloud project, - // it means that the project was modified by someone else. In this case, we should warn - // the user and ask them if they want to overwrite the changes. - const cloudProjectId = fileMetadata.fileIdentifier; - const openedProjectVersion = fileMetadata.version; - const cloudProject: ?CloudProjectWithUserAccessInfo = await getCloudProject( - authenticatedUser, - cloudProjectId - ); - if (!cloudProject) { - await actions.showAlert({ - title: t`Unable to save the project`, - message: t`The project could not be saved. Please try again later.`, - }); - return false; - } - const { currentVersion, committedAt } = cloudProject; - if ( - openedProjectVersion && - currentVersion && // should always be defined. - committedAt && // should always be defined. - currentVersion !== openedProjectVersion - ) { - let lastUsernameWhoModifiedProject = null; - const committedAtDate = new Date(committedAt); - const formattedDate = format(committedAtDate, 'dd-MM-yyyy'); - const formattedTime = format(committedAtDate, 'HH:mm:ss'); - const lastCommittedBy = cloudProject.lastCommittedBy; - if (lastCommittedBy) { - const lastUser = await getUserPublicProfile(lastCommittedBy); - if (lastUser) { - lastUsernameWhoModifiedProject = lastUser.username; - } - } - const answer = await actions.showConfirmation({ - title: t`Project was modified`, - message: lastUsernameWhoModifiedProject - ? t`This project was modified by ${lastUsernameWhoModifiedProject} on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?` - : t`This project was modified by someone else on the ${formattedDate} at ${formattedTime}. Do you want to overwrite their changes?`, - level: 'warning', - confirmButtonLabel: t`Overwrite`, - makeDismissButtonPrimary: true, - }); - - return answer; - } - - return true; - }, }), createResourceOperations: generateGetResourceActions, }: StorageProvider); diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalFileStorageProviderInternalName.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalFileStorageProviderInternalName.js new file mode 100644 index 000000000000..1eee2a30afa7 --- /dev/null +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalFileStorageProviderInternalName.js @@ -0,0 +1,3 @@ +// @flow + +export const localFileStorageProviderInternalName = 'LocalFile'; diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js index bc5542e81fb7..579dc915a634 100644 --- a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/LocalProjectWriter.js @@ -2,10 +2,12 @@ import { t, Trans } from '@lingui/macro'; import * as React from 'react'; import { serializeToJSObject, serializeToJSON } from '../../Utils/Serializer'; +import { serializeToJSObjectInBackground } from '../../Utils/BackgroundSerializer'; import { type FileMetadata, type SaveAsLocation, type SaveAsOptions, + type SaveProjectOptions, } from '../index'; import optionalRequire from '../../Utils/OptionalRequire'; import { @@ -16,6 +18,10 @@ import { import type { MessageDescriptor } from '../../Utils/i18n/MessageDescriptor.flow'; import LocalFolderPicker from '../../UI/LocalFolderPicker'; import SaveAsOptionsDialog from '../SaveAsOptionsDialog'; +import { + type ShowAlertFunction, + type ShowConfirmFunction, +} from '../../UI/Alert/AlertContext'; const fs = optionalRequire('fs-extra'); const path = optionalRequire('path'); @@ -102,12 +108,27 @@ const writeAndCheckFormattedJSONFile = async ( await writeAndCheckFile(content, filePath); }; -const writeProjectFiles = ( +const writeProjectFiles = async ({ + project, + filePath, + projectPath, + useBackgroundSerializer, +}: { project: gdProject, filePath: string, - projectPath: string -): Promise => { - const serializedProjectObject = serializeToJSObject(project); + projectPath: string, + useBackgroundSerializer: boolean, +}): Promise => { + const startTime = Date.now(); + + let serializedProjectObject; + if (useBackgroundSerializer) { + serializedProjectObject = await serializeToJSObjectInBackground(project); + } else { + serializedProjectObject = serializeToJSObject(project); + } + const serializeEndTime = Date.now(); + if (project.isFolderProject()) { const partialObjects = split(serializedProjectObject, { pathSeparator: '/', @@ -140,23 +161,37 @@ const writeProjectFiles = ( }); }); } else { - return writeAndCheckFormattedJSONFile( - serializedProjectObject, - filePath - ).catch(err => { - console.error('Unable to write the project:', err); - throw err; - }); + await writeAndCheckFormattedJSONFile(serializedProjectObject, filePath); } + + console.log( + `[LocalProjectWriter] Project file(s) written in ${Date.now() - + startTime}ms (including ${serializeEndTime - startTime}ms for ${ + useBackgroundSerializer ? 'background' : 'main' + } thread serialization)` + ); }; export const onSaveProject = async ( project: gdProject, - fileMetadata: FileMetadata + fileMetadata: FileMetadata, + saveOptions?: SaveProjectOptions, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} ): Promise<{| wasSaved: boolean, fileMetadata: FileMetadata, |}> => { + const canBeSafelySaved = await canFileMetadataBeSafelySaved( + fileMetadata, + actions + ); + if (!canBeSafelySaved) { + return { wasSaved: false, fileMetadata: fileMetadata }; + } + const filePath = fileMetadata.fileIdentifier; const now = Date.now(); if (!filePath) { @@ -178,7 +213,13 @@ export const onSaveProject = async ( console.warn('Unable to clean project folder before saving project: ', e); } - await writeProjectFiles(project, filePath, projectPath); + await writeProjectFiles({ + project, + filePath, + projectPath, + useBackgroundSerializer: + !!saveOptions && !!saveOptions.useBackgroundSerializer, + }); return { wasSaved: true, fileMetadata: newFileMetadata, @@ -295,7 +336,12 @@ export const onSaveProjectAs = async ( const projectPath = path.dirname(filePath); project.setProjectFile(filePath); - await writeProjectFiles(project, filePath, projectPath); + await writeProjectFiles({ + project, + filePath, + projectPath, + useBackgroundSerializer: false, + }); return { wasSaved: true, fileMetadata: newFileMetadata, @@ -392,7 +438,7 @@ export const renderNewProjectSaveAsLocationChooser = ({ ); }; -export const isTryingToSaveInForbiddenPath = (filePath: string): boolean => { +const isTryingToSaveInForbiddenPath = (filePath: string): boolean => { if (!remote) return false; // This should not happen, but let's be safe. // If the user is saving locally and chose the same location as where the // executable is running, prevent this, as it will be deleted when the app is updated. @@ -401,3 +447,43 @@ export const isTryingToSaveInForbiddenPath = (filePath: string): boolean => { const gdevelopDirectory = path.dirname(exePath); return filePath.startsWith(gdevelopDirectory); }; + +export const canFileMetadataBeSafelySaved = async ( + fileMetadata: FileMetadata, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} +) => { + const path = fileMetadata.fileIdentifier; + if (isTryingToSaveInForbiddenPath(path)) { + await actions.showAlert({ + title: t`Choose another location`, + message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, + }); + } + + // We don't block the save, in case the user wants to save anyway. + return true; +}; + +export const canFileMetadataBeSafelySavedAs = async ( + fileMetadata: FileMetadata, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} +) => { + const path = fileMetadata.fileIdentifier; + if (isTryingToSaveInForbiddenPath(path)) { + await actions.showAlert({ + title: t`Choose another location`, + message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, + }); + + // We block the save as we don't want new projects to be saved there. + return false; + } + + return true; +}; diff --git a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js index d691614a96fa..31606e8c9aba 100644 --- a/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js +++ b/newIDE/app/src/ProjectsStorage/LocalFileStorageProvider/index.js @@ -1,7 +1,7 @@ // @flow import * as React from 'react'; import { Trans, t } from '@lingui/macro'; -import { type StorageProvider, type FileMetadata } from '../index'; +import { type StorageProvider } from '../index'; import { onOpenWithPicker, onOpen, @@ -16,7 +16,7 @@ import { getWriteErrorMessage, renderNewProjectSaveAsLocationChooser, getProjectLocation, - isTryingToSaveInForbiddenPath, + canFileMetadataBeSafelySavedAs, } from './LocalProjectWriter'; import { type AppArguments, @@ -32,18 +32,15 @@ import { scanForNewResources, } from './LocalProjectResourcesHandler'; import { allResourceKindsAndMetadata } from '../../ResourcesList/ResourceSource'; -import { - type ShowAlertFunction, - type ShowConfirmFunction, -} from '../../UI/Alert/AlertContext'; import { setupResourcesWatcher } from './LocalFileResourcesWatcher'; +import { localFileStorageProviderInternalName } from './LocalFileStorageProviderInternalName'; /** * Use the Electron APIs to provide access to the native * file system (with native save/open dialogs). */ export default ({ - internalName: 'LocalFile', + internalName: localFileStorageProviderInternalName, name: t`Your computer`, renderIcon: props => , getFileMetadataFromAppArguments: (appArguments: AppArguments) => { @@ -73,44 +70,7 @@ export default ({ return t`Check that the file exists, that this file is a proper game created with GDevelop and that you have the authorization to open it.`; }, getWriteErrorMessage, - canFileMetadataBeSafelySaved: async ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => { - const path = fileMetadata.fileIdentifier; - if (isTryingToSaveInForbiddenPath(path)) { - await actions.showAlert({ - title: t`Choose another location`, - message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, - }); - } - - // We don't block the save, in case the user wants to save anyway. - return true; - }, - canFileMetadataBeSafelySavedAs: async ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => { - const path = fileMetadata.fileIdentifier; - if (isTryingToSaveInForbiddenPath(path)) { - await actions.showAlert({ - title: t`Choose another location`, - message: t`Your project is saved in the same folder as the application. This folder will be deleted when the application is updated. Please choose another location if you don't want to lose your project.`, - }); - - // We block the save as we don't want new projects to be saved there. - return false; - } - - return true; - }, + canFileMetadataBeSafelySavedAs, }), createResourceOperations: () => ({ project, diff --git a/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js b/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js index 56b9beb49879..93acc9b31148 100644 --- a/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js +++ b/newIDE/app/src/ProjectsStorage/ResourceMover/LocalResourceMover.js @@ -4,10 +4,8 @@ import { type MoveAllProjectResourcesResult, type MoveAllProjectResourcesFunction, } from './index'; -import LocalFileStorageProvider from '../LocalFileStorageProvider'; import { moveUrlResourcesToLocalFiles } from '../LocalFileStorageProvider/LocalFileResourceMover'; import UrlStorageProvider from '../UrlStorageProvider'; -import CloudStorageProvider from '../CloudStorageProvider'; import LocalFileSystem from '../../ExportAndShare/LocalExporters/LocalFileSystem'; import assignIn from 'lodash/assignIn'; import optionalRequire from '../../Utils/OptionalRequire'; @@ -21,6 +19,8 @@ import { import { processByChunk } from '../../Utils/ProcessByChunk'; import { readLocalFileToFile } from '../../Utils/LocalFileUploader'; import { isURL, isBlobURL } from '../../ResourcesList/ResourceUtils'; +import { cloudStorageProviderInternalName } from '../CloudStorageProvider/CloudStoageProviderInternalName'; +import { localFileStorageProviderInternalName } from '../LocalFileStorageProvider/LocalFileStorageProviderInternalName'; const path = optionalRequire('path'); const gd: libGDevelop = global.gd; @@ -164,9 +164,10 @@ export const moveAllLocalResourcesToCloudResources = async ({ const movers: { [string]: MoveAllProjectResourcesFunction, } = { - [`${LocalFileStorageProvider.internalName}=>${ - LocalFileStorageProvider.internalName - }`]: async ({ project, newFileMetadata }: MoveAllProjectResourcesOptions) => { + [`${localFileStorageProviderInternalName}=>${localFileStorageProviderInternalName}`]: async ({ + project, + newFileMetadata, + }: MoveAllProjectResourcesOptions) => { // TODO: Ideally, errors while copying resources should be reported. // TODO: Report progress. const projectPath = path.dirname(newFileMetadata.fileIdentifier); @@ -189,9 +190,12 @@ const movers: { }, // When saving a Cloud project locally, all resources are downloaded (including // the ones on GDevelop Cloud or private game templates). - [`${CloudStorageProvider.internalName}=>${ - LocalFileStorageProvider.internalName - }`]: ({ project, newFileMetadata, onProgress, authenticatedUser }) => + [`${cloudStorageProviderInternalName}=>${localFileStorageProviderInternalName}`]: ({ + project, + newFileMetadata, + onProgress, + authenticatedUser, + }) => moveUrlResourcesToLocalFiles({ project, fileMetadata: newFileMetadata, @@ -204,9 +208,14 @@ const movers: { // for resources). // This is also helpful to download private game templates resources so that // the game can be opened offline. - [`${UrlStorageProvider.internalName}=>${ - LocalFileStorageProvider.internalName - }`]: ({ project, newFileMetadata, onProgress, authenticatedUser }) => + [`${ + UrlStorageProvider.internalName + }=>${localFileStorageProviderInternalName}`]: ({ + project, + newFileMetadata, + onProgress, + authenticatedUser, + }) => moveUrlResourcesToLocalFiles({ project, fileMetadata: newFileMetadata, @@ -217,20 +226,16 @@ const movers: { // Moving to GDevelop "Cloud" storage: // From a local project to a Cloud project, all resources are uploaded. - [`${LocalFileStorageProvider.internalName}=>${ - CloudStorageProvider.internalName - }`]: moveAllLocalResourcesToCloudResources, + [`${localFileStorageProviderInternalName}=>${cloudStorageProviderInternalName}`]: moveAllLocalResourcesToCloudResources, // From a Cloud project to another, resources need to be copied // (unless they are public URLs). - [`${CloudStorageProvider.internalName}=>${ - CloudStorageProvider.internalName - }`]: moveUrlResourcesToCloudProject, + [`${cloudStorageProviderInternalName}=>${cloudStorageProviderInternalName}`]: moveUrlResourcesToCloudProject, // Nothing to move around when going from a project on a public URL // to a cloud project (we could offer an option one day though to download // and upload the URL resources on GDevelop Cloud). - [`${UrlStorageProvider.internalName}=>${ - CloudStorageProvider.internalName - }`]: moveUrlResourcesToCloudProject, + [`${ + UrlStorageProvider.internalName + }=>${cloudStorageProviderInternalName}`]: moveUrlResourcesToCloudProject, }; const LocalResourceMover = { diff --git a/newIDE/app/src/ProjectsStorage/index.js b/newIDE/app/src/ProjectsStorage/index.js index 60e29269283b..d535823084aa 100644 --- a/newIDE/app/src/ProjectsStorage/index.js +++ b/newIDE/app/src/ProjectsStorage/index.js @@ -50,6 +50,12 @@ export type SaveAsLocation = {| // a new location where to save a project to. |}; +export type SaveProjectOptions = {| + previousVersion?: string, + restoredFromVersionId?: string, + useBackgroundSerializer?: boolean, +|}; + export type SaveAsOptions = {| generateNewProjectUuid?: boolean, setProjectNameFromLocation?: boolean, @@ -106,7 +112,11 @@ export type StorageProviderOperations = {| onSaveProject?: ( project: gdProject, fileMetadata: FileMetadata, - options?: {| previousVersion?: string, restoredFromVersionId?: string |} + options?: SaveProjectOptions, + actions: {| + showAlert: ShowAlertFunction, + showConfirmation: ShowConfirmFunction, + |} ) => Promise<{| wasSaved: boolean, fileMetadata: FileMetadata, @@ -133,13 +143,6 @@ export type StorageProviderOperations = {| /** This is the location where the project was saved, or null if not persisted. */ fileMetadata: ?FileMetadata, |}>, - canFileMetadataBeSafelySaved?: ( - fileMetadata: FileMetadata, - actions: {| - showAlert: ShowAlertFunction, - showConfirmation: ShowConfirmFunction, - |} - ) => Promise, canFileMetadataBeSafelySavedAs?: ( fileMetadata: FileMetadata, actions: {| diff --git a/newIDE/app/src/Utils/BackgroundSerializer.js b/newIDE/app/src/Utils/BackgroundSerializer.js new file mode 100644 index 000000000000..32083650b27a --- /dev/null +++ b/newIDE/app/src/Utils/BackgroundSerializer.js @@ -0,0 +1,132 @@ +// @flow + +import VersionMetadata from '../Version/VersionMetadata'; +// Import the worker (handled by worker-loader) +import BackgroundSerializerWorker from './BackgroundSerializer.worker'; + +const gd: libGDevelop = global.gd; + +type BackgroundSerializerWorkerOutMessage = + | {| type: 'DONE', requestId: number, result: any |} + | {| type: 'ERROR', requestId: number, message: string |}; + +let serializerWorker: ?Worker = null; +let nextRequestId = 1; +const pendingRequests: Map< + number, + { resolve: (result: any) => void, reject: (error: Error) => void } +> = new Map(); + +const log = (message: string) => { + console.log(`[BackgroundSerializer] ${message}`); +}; + +const getOrCreateBackgroundSerializerWorker = (): Worker => { + if (serializerWorker) { + return serializerWorker; + } + + // $FlowExpectedError - worker-loader types aren't recognized by Flow + serializerWorker = new BackgroundSerializerWorker(); + + // Set up message handler + serializerWorker.onmessage = (event: MessageEvent) => { + const data: BackgroundSerializerWorkerOutMessage = (event.data: any); + + const pending = pendingRequests.get(data.requestId); + if (!pending) { + console.warn( + `[BackgroundSerializer] Received message for unknown request ID #${ + data.requestId + }.` + ); + return; + } + pendingRequests.delete(data.requestId); + + if (data.type === 'DONE') { + pending.resolve(data.result); + return; + } + pending.reject(new Error(data.message || 'Unknown error')); + }; + + serializerWorker.onerror = error => { + console.error('[BackgroundSerializer] Worker sent an error.', error); + }; + + return serializerWorker; +}; + +const sendMessageToBackgroundSerializerWorker = (message: {| + type: 'SERIALIZE_TO_JSON' | 'SERIALIZE_TO_JS_OBJECT', + binary: Uint8Array, + versionWithHash: string, +|}) => { + const worker = getOrCreateBackgroundSerializerWorker(); + + return new Promise((resolve, reject) => { + const requestId = nextRequestId++; + pendingRequests.set(requestId, { resolve, reject }); + worker.postMessage({ ...message, requestId }); + }); +}; + +export async function serializeInBackground( + type: 'SERIALIZE_TO_JSON' | 'SERIALIZE_TO_JS_OBJECT', + serializable: gdSerializable +): Promise { + const startTime = Date.now(); + let serializedElement = new gd.SerializerElement(); + serializable.serializeTo(serializedElement); + + const serializeToEndTime = Date.now(); + + const binaryPtr = gd.BinarySerializer.createBinarySnapshot(serializedElement); + const binarySize = gd.BinarySerializer.getLastBinarySnapshotSize(); + serializedElement.delete(); + serializedElement = null; + + if (!binaryPtr) { + throw new Error('Failed to create binary snapshot.'); + } + + const binaryView = new Uint8Array(gd.HEAPU8.buffer, binaryPtr, binarySize); + // Copy the buffer out of the WASM heap so it can be transferred. + const binaryBuffer = binaryView.slice(); + + gd.BinarySerializer.freeBinarySnapshot(binaryPtr); + + const binaryBufferEndTime = Date.now(); + log( + `Spent ${binaryBufferEndTime - + startTime}ms on main thread (including ${serializeToEndTime - + startTime}ms for SerializerElement serialization and ${binaryBufferEndTime - + serializeToEndTime}ms for BinaryBuffer preparation).` + ); + + const result = await sendMessageToBackgroundSerializerWorker({ + type, + binary: binaryBuffer, + versionWithHash: VersionMetadata.versionWithHash, + }); + + const workerPromiseEndTime = Date.now(); + log( + `The worker returned in ${workerPromiseEndTime - binaryBufferEndTime}ms.` + ); + + return result; +} + +export const serializeToJSONInBackground = async ( + serializable: gdSerializable +): Promise => { + return serializeInBackground('SERIALIZE_TO_JSON', serializable); +}; + +export const serializeToJSObjectInBackground = async ( + serializable: gdSerializable +): Promise => { + return serializeInBackground('SERIALIZE_TO_JS_OBJECT', serializable); +}; diff --git a/newIDE/app/src/Utils/BackgroundSerializer.worker.js b/newIDE/app/src/Utils/BackgroundSerializer.worker.js new file mode 100644 index 000000000000..e503407d287c --- /dev/null +++ b/newIDE/app/src/Utils/BackgroundSerializer.worker.js @@ -0,0 +1,115 @@ +/* eslint-env worker */ +// @flow + +let modulePromise /*: ?Promise*/ = null; + +const log = (message /*: string */) => { + console.log(`[BackgroundSerializerWorker] ${message}`); +}; + +const getLibGDevelop = (versionWithHash /*: string */) => { + if (modulePromise) return modulePromise; + + modulePromise = new Promise((resolve, reject) => { + try { + const url = `/libGD.js?cache-buster=${versionWithHash}`; + // Load libGD.js in the worker context. + // eslint-disable-next-line no-undef + importScripts(url); + + // eslint-disable-next-line no-undef + // $FlowFixMe + if (typeof initializeGDevelopJs !== 'function') { + reject(new Error('Missing initializeGDevelopJs in worker')); + return; + } + + // eslint-disable-next-line no-undef + initializeGDevelopJs({ + // Override the resolved URL for the .wasm file, + // to ensure a new version is fetched when the version changes. + locateFile: (path /*: string */, prefix /*: string */) => { + // This function is called by Emscripten to locate the .wasm file only. + // As the wasm is at the root of the public folder, we can just return + // the path to the file. + // Plus, on Electron, the prefix seems to be pointing to the root of the + // app.asar archive, which is completely wrong. + return path + `?cache-buster=${versionWithHash}`; + }, + }) + .then(module => { + resolve(module); + }) + .catch(reject); + } catch (error) { + reject(error); + return; + } + }); + + return modulePromise; +}; + +const unserializeBinarySnapshotToJson = ( + gd /*: libGDevelop */, + binary /*: Uint8Array */ +) => { + const binaryArray = + binary instanceof Uint8Array ? binary : new Uint8Array(binary); + const binarySize = binaryArray.byteLength || binaryArray.length; + + // Allocate memory in Emscripten heap and copy binary data + const binaryPtr = gd._malloc(binarySize); + gd.HEAPU8.set(binaryArray, binaryPtr); + + const element = gd.BinarySerializer.deserializeBinarySnapshot( + binaryPtr, + binarySize + ); + + // Free the input buffer + gd._free(binaryPtr); + + if (element.ptr === 0) { + throw new Error('Failed to deserialize binary snapshot.'); + } + + const json = gd.Serializer.toJSON(element); + element.delete(); + return json; +}; + +// eslint-disable-next-line no-restricted-globals +self.onmessage = async (event /*: MessageEvent */) => { + // $FlowExpectedError + const { type, binary, requestId, versionWithHash } = event.data || {}; + + const startTime = Date.now(); + + log(`Request #${requestId} received (${type}).`); + if (type !== 'SERIALIZE_TO_JSON' && type !== 'SERIALIZE_TO_JS_OBJECT') return; + + try { + const gd = await getLibGDevelop(versionWithHash); + + const json = unserializeBinarySnapshotToJson(gd, binary); + const result = type === 'SERIALIZE_TO_JSON' ? json : JSON.parse(json); + + log(`Request #${requestId} done in ${Date.now() - startTime}ms.`); + + // eslint-disable-next-line no-restricted-globals + self.postMessage({ + type: 'DONE', + result, + requestId, + duration: Date.now() - startTime, + }); + } catch (error) { + // eslint-disable-next-line no-restricted-globals + self.postMessage({ + type: 'ERROR', + requestId, + message: error.message, + }); + } +}; diff --git a/newIDE/app/src/Utils/GDevelopServices/Project.js b/newIDE/app/src/Utils/GDevelopServices/Project.js index 363113d731f3..bd55656557c4 100644 --- a/newIDE/app/src/Utils/GDevelopServices/Project.js +++ b/newIDE/app/src/Utils/GDevelopServices/Project.js @@ -298,12 +298,14 @@ export const createCloudProject = async ( export const commitVersion = async ({ authenticatedUser, cloudProjectId, + presignedUrl, zippedProject, previousVersion, restoredFromVersionId, }: { authenticatedUser: AuthenticatedUser, cloudProjectId: string, + presignedUrl: string, zippedProject: Blob, previousVersion?: ?string, restoredFromVersionId?: ?string, @@ -313,12 +315,7 @@ export const commitVersion = async ({ const { uid: userId } = firebaseUser; const authorizationHeader = await getAuthorizationHeader(); - // Get a presigned url to upload a new version (the URL will contain the new version id). - const presignedUrl = await getPresignedUrlForVersionUpload( - authenticatedUser, - cloudProjectId - ); - if (!presignedUrl) return; + const newVersion = getVersionIdFromPath(presignedUrl); // Upload zipped project. await refetchCredentialsForProjectAndRetryIfUnauthorized( @@ -552,7 +549,7 @@ export const deleteCloudProject = async ( }); }; -const getPresignedUrlForVersionUpload = async ( +export const getPresignedUrlForVersionUpload = async ( authenticatedUser: AuthenticatedUser, cloudProjectId: string ): Promise => { diff --git a/newIDE/app/src/Utils/Serializer.js b/newIDE/app/src/Utils/Serializer.js index b026c224ee0e..2ee60041d0c3 100644 --- a/newIDE/app/src/Utils/Serializer.js +++ b/newIDE/app/src/Utils/Serializer.js @@ -1,5 +1,4 @@ // @flow - const gd: libGDevelop = global.gd; /** @@ -19,6 +18,7 @@ export function serializeToJSObject( // JSON.parse + toJSON is 30% faster than gd.Serializer.toJSObject. const json = gd.Serializer.toJSON(serializedElement); + try { const object = JSON.parse(json);