diff --git a/BinaryFormat/BinaryRobloxFile.cs b/BinaryFormat/BinaryRobloxFile.cs index ec278fa..8029cc2 100644 --- a/BinaryFormat/BinaryRobloxFile.cs +++ b/BinaryFormat/BinaryRobloxFile.cs @@ -15,20 +15,22 @@ namespace RobloxFiles // Header Specific public const string MagicHeader = " Chunks = new List(); + internal List ChunksImpl = new List(); + public IReadOnlyList Chunks => ChunksImpl; public override string ToString() => GetType().Name; - public Instance[] Instances; - public INST[] Classes; + public Instance[] Instances { get; internal set; } + public INST[] Classes { get; internal set; } internal META META = null; internal SSTR SSTR = null; + internal SIGN SIGN = null; public bool HasMetadata => (META != null); public Dictionary Metadata => META?.Data; @@ -36,6 +38,9 @@ namespace RobloxFiles public bool HasSharedStrings => (SSTR != null); public IReadOnlyDictionary SharedStrings => SSTR?.Strings; + public bool HasSignatures => (SIGN != null); + public IReadOnlyList Signatures => SIGN?.Signatures; + public BinaryRobloxFile() { Name = "BinaryRobloxFile"; @@ -71,12 +76,10 @@ namespace RobloxFiles { try { - BinaryRobloxFileChunk chunk = new BinaryRobloxFileChunk(reader); - string chunkType = chunk.ChunkType; - + var chunk = new BinaryRobloxFileChunk(reader); IBinaryFileChunk handler = null; - switch (chunkType) + switch (chunk.ChunkType) { case "INST": handler = new INST(); @@ -93,22 +96,27 @@ namespace RobloxFiles case "SSTR": handler = new SSTR(); break; + case "SIGN": + handler = new SIGN(); + break; case "END\0": - Chunks.Add(chunk); + ChunksImpl.Add(chunk); reading = false; break; - default: - Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", chunkType); + case string unhandled: + Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", unhandled); break; + default: break; } if (handler != null) { - using (BinaryRobloxFileReader dataReader = chunk.GetDataReader(this)) - handler.LoadFromReader(dataReader); - chunk.Handler = handler; - Chunks.Add(chunk); + + using (var dataReader = chunk.GetDataReader(this)) + handler.Load(dataReader); + + ChunksImpl.Add(chunk); } } catch (EndOfStreamException) @@ -129,7 +137,7 @@ namespace RobloxFiles { // Clear the existing data. Referent = "-1"; - Chunks.Clear(); + ChunksImpl.Clear(); NumInstances = 0; NumClasses = 0; @@ -148,7 +156,7 @@ namespace RobloxFiles // Write the PROP chunks. foreach (INST inst in Classes) { - Dictionary props = PROP.CollectProperties(writer, inst); + var props = PROP.CollectProperties(writer, inst); foreach (string propName in props.Keys) { @@ -158,26 +166,23 @@ namespace RobloxFiles } // Write the PRNT chunk. - PRNT parents = new PRNT(); + var parents = new PRNT(); writer.SaveChunk(parents); // Write the SSTR chunk. if (HasSharedStrings) - { - var sharedStrings = SSTR.SaveAsChunk(writer); - Chunks.Insert(0, sharedStrings); - } + writer.SaveChunk(SSTR, 0); // Write the META chunk. if (HasMetadata) - { - var metaChunk = META.SaveAsChunk(writer); - Chunks.Insert(0, metaChunk); - } + writer.SaveChunk(META, 0); - // Write the END_ chunk. - var endChunk = writer.WriteEndChunk(); - Chunks.Add(endChunk); + // Write the SIGN chunk. + if (HasSignatures) + writer.SaveChunk(SIGN); + + // Write the END chunk. + writer.WriteChunk("END", ""); } ////////////////////////////////////////////////////////////////////////// diff --git a/BinaryFormat/Chunks/INST.cs b/BinaryFormat/Chunks/INST.cs index 0c948c9..7bde3a6 100644 --- a/BinaryFormat/Chunks/INST.cs +++ b/BinaryFormat/Chunks/INST.cs @@ -16,7 +16,7 @@ namespace RobloxFiles.BinaryFormat.Chunks public override string ToString() => ClassName; - public void LoadFromReader(BinaryRobloxFileReader reader) + public void Load(BinaryRobloxFileReader reader) { BinaryRobloxFile file = reader.File; @@ -59,10 +59,8 @@ namespace RobloxFiles.BinaryFormat.Chunks file.Classes[ClassIndex] = this; } - public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + public void Save(BinaryRobloxFileWriter writer) { - writer.StartWritingChunk(this); - writer.Write(ClassIndex); writer.WriteString(ClassName); @@ -72,7 +70,7 @@ namespace RobloxFiles.BinaryFormat.Chunks if (IsService) { - BinaryRobloxFile file = writer.File; + var file = writer.File; foreach (int instId in InstanceIds) { @@ -80,8 +78,6 @@ namespace RobloxFiles.BinaryFormat.Chunks writer.Write(service.Parent == file); } } - - return writer.FinishWritingChunk(); } } } diff --git a/BinaryFormat/Chunks/META.cs b/BinaryFormat/Chunks/META.cs index 438752f..9506a6c 100644 --- a/BinaryFormat/Chunks/META.cs +++ b/BinaryFormat/Chunks/META.cs @@ -6,7 +6,7 @@ namespace RobloxFiles.BinaryFormat.Chunks { public Dictionary Data = new Dictionary(); - public void LoadFromReader(BinaryRobloxFileReader reader) + public void Load(BinaryRobloxFileReader reader) { BinaryRobloxFile file = reader.File; int numEntries = reader.ReadInt32(); @@ -21,18 +21,15 @@ namespace RobloxFiles.BinaryFormat.Chunks file.META = this; } - public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + public void Save(BinaryRobloxFileWriter writer) { - writer.StartWritingChunk(this); writer.Write(Data.Count); - foreach (var kvPair in Data) + foreach (var pair in Data) { - writer.WriteString(kvPair.Key); - writer.WriteString(kvPair.Value); + writer.WriteString(pair.Key); + writer.WriteString(pair.Value); } - - return writer.FinishWritingChunk(); } } } diff --git a/BinaryFormat/Chunks/PRNT.cs b/BinaryFormat/Chunks/PRNT.cs index 23ba0f6..0803c48 100644 --- a/BinaryFormat/Chunks/PRNT.cs +++ b/BinaryFormat/Chunks/PRNT.cs @@ -1,29 +1,29 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; namespace RobloxFiles.BinaryFormat.Chunks { public class PRNT : IBinaryFileChunk { - public byte Format { get; private set; } - public int NumRelations { get; private set; } + private const byte FORMAT = 0; - public List ChildrenIds { get; private set; } - public List ParentIds { get; private set; } - - public void LoadFromReader(BinaryRobloxFileReader reader) + public void Load(BinaryRobloxFileReader reader) { BinaryRobloxFile file = reader.File; - Format = reader.ReadByte(); - NumRelations = reader.ReadInt32(); + byte format = reader.ReadByte(); + int idCount = reader.ReadInt32(); - ChildrenIds = reader.ReadInstanceIds(NumRelations); - ParentIds = reader.ReadInstanceIds(NumRelations); + if (format != FORMAT) + throw new Exception($"Unexpected PRNT format: {format} (expected {FORMAT}!)"); + + var childIds = reader.ReadInstanceIds(idCount); + var parentIds = reader.ReadInstanceIds(idCount); - for (int i = 0; i < NumRelations; i++) + for (int i = 0; i < idCount; i++) { - int childId = ChildrenIds[i]; - int parentId = ParentIds[i]; + int childId = childIds[i]; + int parentId = parentIds[i]; Instance child = file.Instances[childId]; child.Parent = (parentId >= 0 ? file.Instances[parentId] : file); @@ -31,15 +31,13 @@ namespace RobloxFiles.BinaryFormat.Chunks } } - public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + public void Save(BinaryRobloxFileWriter writer) { - writer.StartWritingChunk(this); + var postInstances = writer.PostInstances; + var idCount = postInstances.Count; - Format = 0; - NumRelations = 0; - - ChildrenIds = new List(); - ParentIds = new List(); + var childIds = new List(); + var parentIds = new List(); foreach (Instance inst in writer.PostInstances) { @@ -51,19 +49,15 @@ namespace RobloxFiles.BinaryFormat.Chunks if (parent != null) parentId = int.Parse(parent.Referent); - ChildrenIds.Add(childId); - ParentIds.Add(parentId); - - NumRelations++; + childIds.Add(childId); + parentIds.Add(parentId); } - writer.Write(Format); - writer.Write(NumRelations); + writer.Write(FORMAT); + writer.Write(idCount); - writer.WriteInstanceIds(ChildrenIds); - writer.WriteInstanceIds(ParentIds); - - return writer.FinishWritingChunk(); + writer.WriteInstanceIds(childIds); + writer.WriteInstanceIds(parentIds); } } } diff --git a/BinaryFormat/Chunks/PROP.cs b/BinaryFormat/Chunks/PROP.cs index f3463cc..8108ce2 100644 --- a/BinaryFormat/Chunks/PROP.cs +++ b/BinaryFormat/Chunks/PROP.cs @@ -29,7 +29,7 @@ namespace RobloxFiles.BinaryFormat.Chunks return $"{Type} {ClassName}.{Name}"; } - public void LoadFromReader(BinaryRobloxFileReader reader) + public void Load(BinaryRobloxFileReader reader) { BinaryRobloxFile file = reader.File; @@ -71,7 +71,6 @@ namespace RobloxFiles.BinaryFormat.Chunks } }); - // Read the property data based on the property type. switch (Type) { case PropertyType.String: @@ -500,13 +499,22 @@ namespace RobloxFiles.BinaryFormat.Chunks readProperties(i => { uint key = SharedKeys[i]; - return file.SharedStrings[key]; }); + break; + case PropertyType.ProtectedString: + readProperties(i => + { + int length = reader.ReadInt32(); + byte[] buffer = reader.ReadBytes(length); + + return new ProtectedString(buffer); + }); + break; default: - Console.WriteLine("Unhandled property type: {0}!", Type); + Console.Error.WriteLine("Unhandled property type: {0}!", Type); break; // } @@ -546,7 +554,7 @@ namespace RobloxFiles.BinaryFormat.Chunks return propMap; } - public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + public void Save(BinaryRobloxFileWriter writer) { BinaryRobloxFile file = writer.File; @@ -567,9 +575,7 @@ namespace RobloxFiles.BinaryFormat.Chunks props.Add(prop); } - writer.StartWritingChunk(this); writer.Write(ClassIndex); - writer.WriteString(Name); writer.Write(TypeId); @@ -996,12 +1002,12 @@ namespace RobloxFiles.BinaryFormat.Chunks props.ForEach(prop => { - SharedString shared = prop.CastValue(); - string key = shared.MD5_Key; + var shared = prop.CastValue(); + string key = shared.Key; if (!sstr.Lookup.ContainsKey(key)) { - uint id = (uint)(sstr.NumHashes++); + uint id = (uint)(sstr.Lookup.Count); sstr.Strings.Add(id, shared); sstr.Lookup.Add(key, id); } @@ -1012,10 +1018,19 @@ namespace RobloxFiles.BinaryFormat.Chunks writer.WriteInterleaved(sharedKeys); break; - // + case PropertyType.ProtectedString: + props.ForEach(prop => + { + var protect = prop.CastValue(); + byte[] buffer = protect.RawBuffer; + + writer.Write(buffer.Length); + writer.Write(buffer); + }); + + break; + default: break; } - - return writer.FinishWritingChunk(); } } } diff --git a/BinaryFormat/Chunks/SIGN.cs b/BinaryFormat/Chunks/SIGN.cs new file mode 100644 index 0000000..7c0a51f --- /dev/null +++ b/BinaryFormat/Chunks/SIGN.cs @@ -0,0 +1,56 @@ +namespace RobloxFiles.BinaryFormat.Chunks +{ + public struct Signature + { + public int Version; + public long Id; + + public int Length; + public byte[] Data; + } + + public class SIGN : IBinaryFileChunk + { + public Signature[] Signatures; + + public void Load(BinaryRobloxFileReader reader) + { + int numSignatures = reader.ReadInt32(); + Signatures = new Signature[numSignatures]; + + for (int i = 0; i < numSignatures; i++) + { + var signature = new Signature + { + Version = reader.ReadInt32(), + Id = reader.ReadInt64(), + + Length = reader.ReadInt32(), + }; + + signature.Data = reader.ReadBytes(signature.Length); + Signatures[i] = signature; + } + + var file = reader.File; + file.SIGN = this; + } + + public void Save(BinaryRobloxFileWriter writer) + { + writer.Write(Signatures.Length); + + for (int i = 0; i < Signatures.Length; i++) + { + var signature = Signatures[i]; + signature.Length = signature.Data.Length; + + writer.Write(signature.Version); + writer.Write(signature.Id); + + writer.Write(signature.Length); + writer.Write(signature.Data); + } + } + } +} diff --git a/BinaryFormat/Chunks/SSTR.cs b/BinaryFormat/Chunks/SSTR.cs index 32c5bee..80ac007 100644 --- a/BinaryFormat/Chunks/SSTR.cs +++ b/BinaryFormat/Chunks/SSTR.cs @@ -6,56 +6,54 @@ namespace RobloxFiles.BinaryFormat.Chunks { public class SSTR : IBinaryFileChunk { - public int Version; - public int NumHashes; + private const int FORMAT = 0; - public Dictionary Lookup = new Dictionary(); - public Dictionary Strings = new Dictionary(); + internal Dictionary Lookup = new Dictionary(); + internal Dictionary Strings = new Dictionary(); - public void LoadFromReader(BinaryRobloxFileReader reader) + public void Load(BinaryRobloxFileReader reader) { BinaryRobloxFile file = reader.File; - Version = reader.ReadInt32(); - NumHashes = reader.ReadInt32(); + int format = reader.ReadInt32(); + int numHashes = reader.ReadInt32(); - for (uint id = 0; id < NumHashes; id++) + if (format != FORMAT) + throw new Exception($"Unexpected SSTR format: {format} (expected {FORMAT}!)"); + + for (uint id = 0; id < numHashes; id++) { - byte[] md5 = reader.ReadBytes(16); - - int length = reader.ReadInt32(); - byte[] data = reader.ReadBytes(length); + byte[] hash = reader.ReadBytes(16); + string key = Convert.ToBase64String(hash); + byte[] data = reader.ReadBuffer(); SharedString value = SharedString.FromBuffer(data); - Lookup.Add(value.MD5_Key, id); + + Lookup.Add(key, id); Strings.Add(id, value); } file.SSTR = this; } - public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + public void Save(BinaryRobloxFileWriter writer) { - writer.StartWritingChunk(this); - - writer.Write(Version); - writer.Write(NumHashes); + writer.Write(FORMAT); + writer.Write(Lookup.Count); foreach (var pair in Lookup) { string key = pair.Key; - byte[] md5 = Convert.FromBase64String(key); - writer.Write(md5); + byte[] hash = Convert.FromBase64String(key); + writer.Write(hash); SharedString value = Strings[pair.Value]; - byte[] buffer = SharedString.FindRecord(value.MD5_Key); + byte[] buffer = SharedString.Find(value.Key); writer.Write(buffer.Length); writer.Write(buffer); } - - return writer.FinishWritingChunk(); } } } diff --git a/BinaryFormat/IO/BinaryFileWriter.cs b/BinaryFormat/IO/BinaryFileWriter.cs index a70c253..f397900 100644 --- a/BinaryFormat/IO/BinaryFileWriter.cs +++ b/BinaryFormat/IO/BinaryFileWriter.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -18,14 +17,16 @@ namespace RobloxFiles.BinaryFormat public string ChunkType { get; private set; } public long ChunkStart { get; private set; } - public Dictionary ClassMap; public readonly BinaryRobloxFile File; + // Dictionary mapping ClassNames to their INST chunks. + private readonly Dictionary ClassMap; + // Instances in parent->child order - public List Instances; + private readonly List Instances; // Instances in child->parent order - public List PostInstances; + internal List PostInstances { get; private set; } public BinaryRobloxFileWriter(BinaryRobloxFile file, Stream workBuffer = null) : base(workBuffer ?? new MemoryStream()) { @@ -104,13 +105,13 @@ namespace RobloxFiles.BinaryFormat } // Encodes an int for an interleaved buffer. - private int EncodeInt(int value) + private static int EncodeInt(int value) { return (value << 1) ^ (value >> 31); } // Encodes a float for an interleaved buffer. - private float EncodeFloat(float value) + private static float EncodeFloat(float value) { byte[] buffer = BitConverter.GetBytes(value); uint bits = BitConverter.ToUInt32(buffer, 0); @@ -171,9 +172,9 @@ namespace RobloxFiles.BinaryFormat Instances.Add(instance); string className = instance.ClassName; - INST inst = null; + INST inst; - if (!ClassMap.ContainsKey(className)) + if (!ClassMap.TryGetValue(className, out inst)) { inst = new INST() { @@ -184,11 +185,7 @@ namespace RobloxFiles.BinaryFormat ClassMap.Add(className, inst); } - else - { - inst = ClassMap[className]; - } - + inst.NumInstances++; inst.InstanceIds.Add(instId); @@ -222,7 +219,7 @@ namespace RobloxFiles.BinaryFormat } // Marks that we are writing a chunk. - public bool StartWritingChunk(string chunkType) + private bool StartWritingChunk(string chunkType) { if (chunkType.Length != 4) throw new Exception("BinaryFileWriter.StartWritingChunk - ChunkType length should be 4!"); @@ -241,7 +238,7 @@ namespace RobloxFiles.BinaryFormat } // Marks that we are writing a chunk. - public bool StartWritingChunk(IBinaryFileChunk chunk) + private bool StartWritingChunk(IBinaryFileChunk chunk) { if (!WritingChunk) { @@ -257,7 +254,7 @@ namespace RobloxFiles.BinaryFormat } // Compresses the data that was written into a BinaryRobloxFileChunk and writes it. - public BinaryRobloxFileChunk FinishWritingChunk(bool compress = true) + private BinaryRobloxFileChunk FinishWritingChunk(bool compress = true) { if (!WritingChunk) throw new Exception($"BinaryRobloxFileWriter: Cannot finish writing a chunk without starting one!"); @@ -283,18 +280,36 @@ namespace RobloxFiles.BinaryFormat return chunk; } - public void SaveChunk(IBinaryFileChunk handler) + internal BinaryRobloxFileChunk SaveChunk(IBinaryFileChunk handler, int insertPos = -1) { - var chunk = handler.SaveAsChunk(this); - File.Chunks.Add(chunk); + StartWritingChunk(handler); + handler.Save(this); + + var chunk = FinishWritingChunk(); + + if (insertPos >= 0) + File.ChunksImpl.Insert(insertPos, chunk); + else + File.ChunksImpl.Add(chunk); + + return chunk; } - public BinaryRobloxFileChunk WriteEndChunk() + internal BinaryRobloxFileChunk WriteChunk(string chunkType, string content, bool compress = false) { - StartWritingChunk("END\0"); - WriteString("", true); + if (chunkType.Length > 4) + chunkType = chunkType.Substring(0, 4); - return FinishWritingChunk(false); + while (chunkType.Length < 4) + chunkType += '\0'; + + StartWritingChunk(chunkType); + WriteString(content); + + var chunk = FinishWritingChunk(compress); + File.ChunksImpl.Add(chunk); + + return chunk; } } } diff --git a/DataTypes/ProtectedString.cs b/DataTypes/ProtectedString.cs index 3b6bc57..97e763b 100644 --- a/DataTypes/ProtectedString.cs +++ b/DataTypes/ProtectedString.cs @@ -1,27 +1,54 @@ -namespace RobloxFiles.DataTypes +using System.Text; + +namespace RobloxFiles.DataTypes { /// - /// ProtectedString is a type used by some of the XML properties. - /// Here, it only exists as a wrapper class for strings. + /// ProtectedString is a type used by the Source property of scripts. + /// If constructed as an array of bytes, it's assumed to be compiled byte-code. /// public class ProtectedString { - public readonly string ProtectedValue; - public override string ToString() => ProtectedValue; + public readonly bool IsCompiled; + public readonly byte[] RawBuffer; + + public override string ToString() + { + if (IsCompiled) + return $"byte[{RawBuffer.Length}]"; + + return Encoding.UTF8.GetString(RawBuffer); + } public ProtectedString(string value) { - ProtectedValue = value; + IsCompiled = false; + RawBuffer = Encoding.UTF8.GetBytes(value); + } + + public ProtectedString(byte[] compiled) + { + IsCompiled = true; + RawBuffer = compiled; } public static implicit operator string(ProtectedString protectedString) { - return protectedString.ProtectedValue; + return Encoding.UTF8.GetString(protectedString.RawBuffer); } public static implicit operator ProtectedString(string value) { return new ProtectedString(value); } + + public static implicit operator byte[](ProtectedString protectedString) + { + return protectedString.RawBuffer; + } + + public static implicit operator ProtectedString(byte[] value) + { + return new ProtectedString(value); + } } } diff --git a/DataTypes/SharedString.cs b/DataTypes/SharedString.cs index 98a17ae..e924d27 100644 --- a/DataTypes/SharedString.cs +++ b/DataTypes/SharedString.cs @@ -1,43 +1,58 @@ using System; -using System.Collections.Generic; using System.Text; -using System.Security.Cryptography; +using System.Collections.Generic; +using Konscious.Security.Cryptography; namespace RobloxFiles.DataTypes { + // SharedString is a datatype that takes a sequence of bytes and stores it in a + // lookup table that is shared by the entire file. It originally used MD5 for the + // hashing, but Roblox now uses Blake2B to avoid the obvious problems with using MD5. + + // In practice the value of a SharedString does not have to match the hash of the + // data it represents, it just needs to be distinct and MUST be 16 bytes long. + // The XML format still uses 'md5' as its attribute key to the lookup table. + public class SharedString { - private static Dictionary Records = new Dictionary(); - public readonly string MD5_Key; + private static Dictionary Lookup = new Dictionary(); + public string Key { get; internal set; } + public string ComputedKey { get; internal set; } - public byte[] SharedValue => FindRecord(MD5_Key); - public override string ToString() => $"MD5 Key: {MD5_Key}"; + public byte[] SharedValue => Find(ComputedKey ?? Key); + public override string ToString() => $"Key: {ComputedKey ?? Key}"; - internal SharedString(string md5) + internal SharedString(string key) { - MD5_Key = md5; + Key = key; + } + + internal static void Register(string key, byte[] buffer) + { + Lookup.Add(key, buffer); } private SharedString(byte[] buffer) { - using (MD5 md5 = MD5.Create()) + using (HMACBlake2B blake2B = new HMACBlake2B(16 * 8)) { - byte[] hash = md5.ComputeHash(buffer); - MD5_Key = Convert.ToBase64String(hash); + byte[] hash = blake2B.ComputeHash(buffer); + ComputedKey = Convert.ToBase64String(hash); + Key = ComputedKey; } - if (Records.ContainsKey(MD5_Key)) + if (Lookup.ContainsKey(ComputedKey)) return; - Records.Add(MD5_Key, buffer); + Register(ComputedKey, buffer); } - public static byte[] FindRecord(string key) + public static byte[] Find(string key) { byte[] result = null; - if (Records.ContainsKey(key)) - result = Records[key]; + if (Lookup.ContainsKey(key)) + result = Lookup[key]; return result; } diff --git a/Interfaces/IBinaryFileChunk.cs b/Interfaces/IBinaryFileChunk.cs index 6011ecc..0a02908 100644 --- a/Interfaces/IBinaryFileChunk.cs +++ b/Interfaces/IBinaryFileChunk.cs @@ -2,7 +2,7 @@ { public interface IBinaryFileChunk { - void LoadFromReader(BinaryRobloxFileReader reader); - BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer); + void Load(BinaryRobloxFileReader reader); + void Save(BinaryRobloxFileWriter writer); } } diff --git a/RobloxFile.cs b/RobloxFile.cs index 051b84e..852cd09 100644 --- a/RobloxFile.cs +++ b/RobloxFile.cs @@ -6,17 +6,23 @@ using System.Threading.Tasks; namespace RobloxFiles { /// - /// Represents a loaded *.rbxl/*.rbxm Roblox file. - /// The contents of the RobloxFile are stored as its children. + /// Represents a loaded Roblox place/model file. + /// RobloxFile is an Instance and its children are the contents of the file. /// public abstract class RobloxFile : Instance { protected abstract void ReadFile(byte[] buffer); + + /// + /// Saves this RobloxFile to the provided stream. + /// + /// The stream to save to. public abstract void Save(Stream stream); /// /// Opens a RobloxFile using the provided buffer. /// + /// The opened RobloxFile. public static RobloxFile Open(byte[] buffer) { if (buffer.Length > 14) @@ -38,11 +44,12 @@ namespace RobloxFiles throw new Exception("Unrecognized header!"); } - + /// /// Opens a Roblox file by reading from a provided Stream. /// /// The stream to read the Roblox file from. + /// The opened RobloxFile. public static RobloxFile Open(Stream stream) { byte[] buffer; @@ -60,6 +67,7 @@ namespace RobloxFiles /// Opens a Roblox file from a provided file path. /// /// A path to a Roblox file to be opened. + /// The opened RobloxFile. public static RobloxFile Open(string filePath) { byte[] buffer = File.ReadAllBytes(filePath); @@ -70,6 +78,7 @@ namespace RobloxFiles /// Creates and runs a Task to open a Roblox file from a byte sequence that represents the file. /// /// A byte sequence that represents the file. + /// A task which will complete once the file is opened with the resulting RobloxFile. public static Task OpenAsync(byte[] buffer) { return Task.Run(() => Open(buffer)); @@ -79,6 +88,7 @@ namespace RobloxFiles /// Creates and runs a Task to open a Roblox file using a provided Stream. /// /// The stream to read the Roblox file from. + /// A task which will complete once the file is opened with the resulting RobloxFile. public static Task OpenAsync(Stream stream) { return Task.Run(() => Open(stream)); @@ -88,9 +98,42 @@ namespace RobloxFiles /// Opens a Roblox file from a provided file path. /// /// A path to a Roblox file to be opened. + /// A task which will complete once the file is opened with the resulting RobloxFile. public static Task OpenAsync(string filePath) { return Task.Run(() => Open(filePath)); } + + /// + /// Saves this RobloxFile to the provided file path. + /// + /// A path to where the file should be saved. + public void Save(string filePath) + { + using (FileStream stream = File.OpenWrite(filePath)) + { + Save(stream); + } + } + + /// + /// Asynchronously saves this RobloxFile to the provided stream. + /// + /// The stream to save to. + /// A task which will complete upon the save's completion. + public Task SaveAsync(Stream stream) + { + return Task.Run(() => Save(stream)); + } + + /// + /// Asynchronously saves this RobloxFile to the provided file path. + /// + /// A path to where the file should be saved. + /// A task which will complete upon the save's completion. + public Task SaveAsync(string filePath) + { + return Task.Run(() => Save(filePath)); + } } } diff --git a/RobloxFileFormat.csproj b/RobloxFileFormat.csproj index 2003c35..8d44611 100644 --- a/RobloxFileFormat.csproj +++ b/RobloxFileFormat.csproj @@ -1,6 +1,6 @@  - + Debug @@ -52,9 +52,11 @@ - - packages\Costura.Fody.3.3.3\lib\net40\Costura.dll - True + + packages\Costura.Fody.4.1.0\lib\net40\Costura.dll + + + packages\Konscious.Security.Cryptography.Blake2.1.0.9\lib\net46\Konscious.Security.Cryptography.Blake2.dll packages\lz4net.1.0.15.93\lib\net4-client\LZ4.dll @@ -62,6 +64,10 @@ + + + packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll + @@ -76,6 +82,7 @@ + @@ -88,7 +95,6 @@ - @@ -167,24 +173,18 @@ - - Always - - - Always - copy /y $(TargetPath) $(ProjectDir)$(TargetFileName) - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - + + + \ No newline at end of file diff --git a/RobloxFileFormat.dll b/RobloxFileFormat.dll index 16a4ea6..8931291 100644 Binary files a/RobloxFileFormat.dll and b/RobloxFileFormat.dll differ diff --git a/RobloxFileFormat.sln b/RobloxFileFormat.sln index 3907fb4..aceb440 100644 --- a/RobloxFileFormat.sln +++ b/RobloxFileFormat.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 16.0.29920.165 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobloxFileFormat", "RobloxFileFormat.csproj", "{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobloxFileFormat.UnitTest", "UnitTest\RobloxFileFormat.UnitTest.csproj", "{6BCA31B2-58D8-4689-9929-88E16040BF29}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -15,6 +17,10 @@ Global {CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Debug|Any CPU.Build.0 = Debug|Any CPU {CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.ActiveCfg = Release|Any CPU {CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.Build.0 = Release|Any CPU + {6BCA31B2-58D8-4689-9929-88E16040BF29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6BCA31B2-58D8-4689-9929-88E16040BF29}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6BCA31B2-58D8-4689-9929-88E16040BF29}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6BCA31B2-58D8-4689-9929-88E16040BF29}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Tree/Attributes.cs b/Tree/Attributes.cs index 12811e9..ef86b4a 100644 --- a/Tree/Attributes.cs +++ b/Tree/Attributes.cs @@ -54,7 +54,7 @@ namespace RobloxFiles } internal BinaryReader reader; - internal BinaryWriter writer; + // internal BinaryWriter writer; internal int readInt() => reader.ReadInt32(); internal byte readByte() => reader.ReadByte(); diff --git a/Tree/Instance.cs b/Tree/Instance.cs index 3a9fa4a..0f6d07c 100644 --- a/Tree/Instance.cs +++ b/Tree/Instance.cs @@ -520,12 +520,12 @@ namespace RobloxFiles continue; PropertyType propType = PropertyType.Unknown; - - if (fieldType.IsEnum) - propType = PropertyType.Enum; - else if (Property.Types.ContainsKey(fieldType)) + + if (Property.Types.ContainsKey(fieldType)) propType = Property.Types[fieldType]; - + else if (fieldType.IsEnum) + propType = PropertyType.Enum; + if (propType != PropertyType.Unknown) { if (fieldName.EndsWith("_")) diff --git a/Tree/Property.cs b/Tree/Property.cs index da40774..fedfb54 100644 --- a/Tree/Property.cs +++ b/Tree/Property.cs @@ -42,6 +42,7 @@ namespace RobloxFiles Color3uint8, Int64, SharedString, + ProtectedString } public class Property @@ -60,6 +61,7 @@ namespace RobloxFiles internal static BindingFlags BindingFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.FlattenHierarchy | BindingFlags.IgnoreCase; internal static MemberTypes FieldOrProperty = MemberTypes.Field | MemberTypes.Property; + // !! FIXME: Map typeof(ProtectedString) to PropertyType.ProtectedString when binary files are allowed to read it. public static readonly IReadOnlyDictionary Types = new Dictionary() { { typeof(Axes), PropertyType.Axes }, @@ -92,24 +94,30 @@ namespace RobloxFiles { typeof(ColorSequence), PropertyType.ColorSequence }, { typeof(NumberSequence), PropertyType.NumberSequence }, - { typeof(ProtectedString), PropertyType.String }, + { typeof(ProtectedString), PropertyType.String }, { typeof(PhysicalProperties), PropertyType.PhysicalProperties }, }; private void ImproviseRawBuffer() { - if (RawValue is byte[]) - { - RawBuffer = RawValue as byte[]; - return; - } - else if (RawValue is SharedString) + if (RawValue is SharedString) { var sharedString = CastValue(); RawBuffer = sharedString.SharedValue; return; } - + else if (RawValue is ProtectedString) + { + var protectedString = CastValue(); + RawBuffer = protectedString.RawBuffer; + return; + } + else if (RawValue is byte[]) + { + RawBuffer = RawValue as byte[]; + return; + } + switch (Type) { case PropertyType.Int: @@ -127,7 +135,7 @@ namespace RobloxFiles case PropertyType.Double: RawBuffer = BitConverter.GetBytes((double)Value); break; - // + default: break; } } @@ -142,8 +150,12 @@ namespace RobloxFiles if (typeName == Name) { - FieldInfo directField = instType.GetField(typeName, BindingFlags.DeclaredOnly); - + FieldInfo directField = instType + .GetFields() + .Where(field => field.Name.StartsWith(Name)) + .Where(field => field.DeclaringType == instType) + .FirstOrDefault(); + if (directField != null) { var implicitName = Name + '_'; diff --git a/LibTest/Binary.rbxl b/UnitTest/Files/Binary.rbxl similarity index 100% rename from LibTest/Binary.rbxl rename to UnitTest/Files/Binary.rbxl diff --git a/LibTest/Xml.rbxlx b/UnitTest/Files/Xml.rbxlx similarity index 100% rename from LibTest/Xml.rbxlx rename to UnitTest/Files/Xml.rbxlx diff --git a/LibTest/Program.cs b/UnitTest/Program.cs similarity index 54% rename from LibTest/Program.cs rename to UnitTest/Program.cs index 92d7b3b..5bc14ab 100644 --- a/LibTest/Program.cs +++ b/UnitTest/Program.cs @@ -2,19 +2,59 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Text.RegularExpressions; using RobloxFiles.DataTypes; -namespace RobloxFiles +namespace RobloxFiles.UnitTest { - // If the solution is built as an exe, this class is - // used to drive some basic testing of the library. - - internal static class Program + static class Program { const string pattern = "\\d+$"; + static void PrintTreeImpl(Instance inst, int stack = 0) + { + string padding = ""; + string extension = ""; + + for (int i = 0; i < stack; i++) + padding += '\t'; + + switch (inst.ClassName) + { + case "Script": + extension = ".server.lua"; + break; + case "LocalScript": + extension = ".client.lua"; + break; + case "ModuleScript": + extension = ".lua"; + break; + default: break; + } + + Console.WriteLine($"{padding}{inst.Name}{extension}"); + + var children = inst + .GetChildren() + .ToList(); + + children.ForEach(child => PrintTreeImpl(child, stack + 1)); + } + + static void PrintTree(string path) + { + Console.WriteLine("Opening file..."); + RobloxFile target = RobloxFile.Open(path); + + foreach (Instance child in target.GetChildren()) + PrintTreeImpl(child); + + Debugger.Break(); + } + static void CountAssets(string path) { Console.WriteLine("Opening file..."); @@ -23,6 +63,14 @@ namespace RobloxFiles var workspace = target.FindFirstChildOfClass(); var assets = new HashSet(); + if (workspace == null) + { + Console.WriteLine("No workspace found!"); + Debugger.Break(); + + return; + } + foreach (Instance inst in workspace.GetDescendants()) { var instPath = inst.GetFullName(); @@ -63,22 +111,19 @@ namespace RobloxFiles if (args.Length > 0) { string path = args[0]; - CountAssets(path); + PrintTree(path); } else { - RobloxFile bin = RobloxFile.Open(@"LibTest\Binary.rbxl"); - RobloxFile xml = RobloxFile.Open(@"LibTest\Xml.rbxlx"); + RobloxFile bin = RobloxFile.Open(@"Files\Binary.rbxl"); + RobloxFile xml = RobloxFile.Open(@"Files\Xml.rbxlx"); Console.WriteLine("Files opened! Pausing execution for debugger analysis..."); Debugger.Break(); - using (FileStream binStream = File.OpenWrite(@"LibTest\Binary_SaveTest.rbxl")) - bin.Save(binStream); - - using (FileStream xmlStream = File.OpenWrite(@"LibTest\Xml_SaveTest.rbxlx")) - xml.Save(xmlStream); - + bin.Save(@"Files\Binary_SaveTest.rbxl"); + xml.Save(@"Files\Xml_SaveTest.rbxlx"); + Console.WriteLine("Files saved! Pausing execution for debugger analysis..."); Debugger.Break(); } diff --git a/UnitTest/RobloxFileFormat.UnitTest.csproj b/UnitTest/RobloxFileFormat.UnitTest.csproj new file mode 100644 index 0000000..49947c4 --- /dev/null +++ b/UnitTest/RobloxFileFormat.UnitTest.csproj @@ -0,0 +1,31 @@ + + + + WinExe + netcoreapp3.1 + + + + + + + + + + + Always + + + Always + + + + + + + + + + + + diff --git a/Utility/Formatting.cs b/Utility/Formatting.cs index aea7fb9..c6a9fd6 100644 --- a/Utility/Formatting.cs +++ b/Utility/Formatting.cs @@ -115,14 +115,18 @@ internal static class Formatting return Math.Abs(a - b) < epsilon; } + public static byte[] ReadBuffer(this BinaryReader reader) + { + int len = reader.ReadInt32(); + return reader.ReadBytes(len); + } + public static string ReadString(this BinaryReader reader, bool useIntLength) { if (!useIntLength) return reader.ReadString(); - int len = reader.ReadInt32(); - byte[] buffer = reader.ReadBytes(len); - + byte[] buffer = reader.ReadBuffer(); return Encoding.UTF8.GetString(buffer); } } \ No newline at end of file diff --git a/XmlFormat/IO/XmlFileReader.cs b/XmlFormat/IO/XmlFileReader.cs index aa1e908..edd070c 100644 --- a/XmlFormat/IO/XmlFileReader.cs +++ b/XmlFormat/IO/XmlFileReader.cs @@ -11,7 +11,7 @@ namespace RobloxFiles.XmlFormat { var errorHandler = new Func((message) => { - string contents = $"XmlDataReader.{label}: {message}"; + string contents = $"XmlRobloxFileReader.{label}: {message}"; return new Exception(contents); }); @@ -29,19 +29,25 @@ namespace RobloxFiles.XmlFormat { if (sharedString.Name == "SharedString") { - XmlNode md5Node = sharedString.Attributes.GetNamedItem("md5"); + XmlNode hashNode = sharedString.Attributes.GetNamedItem("md5"); - if (md5Node == null) + if (hashNode == null) throw error("Got a SharedString without an 'md5' attribute!"); - string key = md5Node.InnerText; + string key = hashNode.InnerText; string value = sharedString.InnerText.Replace("\n", ""); - byte[] buffer = Convert.FromBase64String(value); - SharedString record = SharedString.FromBase64(value); + byte[] hash = Convert.FromBase64String(key); + var record = SharedString.FromBase64(value); - if (record.MD5_Key != key) - throw error("The provided md5 hash did not match with the md5 hash computed for the value!"); + if (hash.Length != 16) + throw error($"SharedString base64 key '{key}' must decode to byte[16]!"); + + if (key != record.Key) + { + SharedString.Register(key, record.SharedValue); + record.Key = key; + } file.SharedStrings.Add(key); } diff --git a/XmlFormat/IO/XmlFileWriter.cs b/XmlFormat/IO/XmlFileWriter.cs index 504a5d6..14249b3 100644 --- a/XmlFormat/IO/XmlFileWriter.cs +++ b/XmlFormat/IO/XmlFileWriter.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Text; using System.Xml; @@ -47,6 +48,9 @@ namespace RobloxFiles.XmlFormat public static XmlNode WriteProperty(Property prop, XmlDocument doc, XmlRobloxFile file) { + if (prop.Name == "Archivable") + return null; + string propType = prop.XmlToken; if (propType == null) @@ -78,6 +82,7 @@ namespace RobloxFiles.XmlFormat case PropertyType.String: propType = (prop.HasRawBuffer ? "BinaryString" : "string"); break; + default: break; } } @@ -89,6 +94,19 @@ namespace RobloxFiles.XmlFormat return null; } + if (prop.Type == PropertyType.SharedString) + { + SharedString value = prop.CastValue(); + + if (value.ComputedKey == null) + { + var newShared = SharedString.FromBuffer(value.SharedValue); + value.Key = newShared.ComputedKey; + } + + file.SharedStrings.Add(value.Key); + } + XmlElement propElement = doc.CreateElement(propType); propElement.SetAttribute("name", prop.Name); @@ -102,12 +120,6 @@ namespace RobloxFiles.XmlFormat propNode = newNode; } - if (prop.Type == PropertyType.SharedString) - { - SharedString value = prop.CastValue(); - file.SharedStrings.Add(value.MD5_Key); - } - return propNode; } @@ -124,11 +136,19 @@ namespace RobloxFiles.XmlFormat instNode.AppendChild(propsNode); var props = instance.RefreshProperties(); + + var orderedKeys = props + .OrderBy(pair => pair.Key) + .Select(pair => pair.Key); - foreach (string propName in props.Keys) + foreach (string propName in orderedKeys) { Property prop = props[propName]; XmlNode propNode = WriteProperty(prop, doc, file); + + if (propNode == null) + continue; + propsNode.AppendChild(propNode); } @@ -151,12 +171,12 @@ namespace RobloxFiles.XmlFormat var binaryWriter = XmlPropertyTokens.GetHandler(); var binaryBuffer = new Property("SharedString", PropertyType.String); - foreach (string md5 in file.SharedStrings) + foreach (string key in file.SharedStrings) { XmlElement sharedString = doc.CreateElement("SharedString"); - sharedString.SetAttribute("md5", md5); + sharedString.SetAttribute("md5", key); - binaryBuffer.RawBuffer = SharedString.FindRecord(md5); + binaryBuffer.RawBuffer = SharedString.Find(key); binaryWriter.WriteProperty(binaryBuffer, doc, sharedString); sharedStrings.AppendChild(sharedString); diff --git a/XmlFormat/Tokens/ProtectedString.cs b/XmlFormat/Tokens/ProtectedString.cs index 833df8b..662c423 100644 --- a/XmlFormat/Tokens/ProtectedString.cs +++ b/XmlFormat/Tokens/ProtectedString.cs @@ -1,4 +1,5 @@ -using System.Xml; +using System.Text; +using System.Xml; using RobloxFiles.DataTypes; namespace RobloxFiles.XmlFormat.PropertyTokens @@ -10,7 +11,7 @@ namespace RobloxFiles.XmlFormat.PropertyTokens public bool ReadProperty(Property prop, XmlNode token) { ProtectedString contents = token.InnerText; - prop.Type = PropertyType.String; + prop.Type = PropertyType.ProtectedString; prop.Value = contents; return true; @@ -18,16 +19,26 @@ namespace RobloxFiles.XmlFormat.PropertyTokens public void WriteProperty(Property prop, XmlDocument doc, XmlNode node) { - string value = prop.CastValue(); + ProtectedString value = prop.CastValue(); - if (value.Contains("\r") || value.Contains("\n")) + if (value.IsCompiled) { - XmlCDataSection cdata = doc.CreateCDataSection(value); - node.AppendChild(cdata); + var binary = XmlPropertyTokens.GetHandler(); + binary.WriteProperty(prop, doc, node); } else { - node.InnerText = value; + string contents = Encoding.UTF8.GetString(value.RawBuffer); + + if (contents.Contains("\r") || contents.Contains("\n")) + { + XmlCDataSection cdata = doc.CreateCDataSection(contents); + node.AppendChild(cdata); + } + else + { + node.InnerText = contents; + } } } } diff --git a/XmlFormat/Tokens/SharedString.cs b/XmlFormat/Tokens/SharedString.cs index 942cc0a..ff210d4 100644 --- a/XmlFormat/Tokens/SharedString.cs +++ b/XmlFormat/Tokens/SharedString.cs @@ -9,17 +9,25 @@ namespace RobloxFiles.XmlFormat.PropertyTokens public bool ReadProperty(Property prop, XmlNode token) { - string md5 = token.InnerText; + string key = token.InnerText; prop.Type = PropertyType.SharedString; - prop.Value = new SharedString(md5); + prop.Value = new SharedString(key); return true; } public void WriteProperty(Property prop, XmlDocument doc, XmlNode node) { - SharedString value = prop.CastValue(); - node.InnerText = value.MD5_Key; + var value = prop.CastValue(); + string key = value.Key; + + if (value.ComputedKey == null) + { + var newShared = SharedString.FromBuffer(value.SharedValue); + key = newShared.ComputedKey; + } + + node.InnerText = key; } } } diff --git a/XmlFormat/XmlRobloxFile.cs b/XmlFormat/XmlRobloxFile.cs index 917250f..759bcd4 100644 --- a/XmlFormat/XmlRobloxFile.cs +++ b/XmlFormat/XmlRobloxFile.cs @@ -103,7 +103,7 @@ namespace RobloxFiles foreach (Property sharedProp in sharedProps) { SharedString shared = sharedProp.CastValue(); - SharedStrings.Add(shared.MD5_Key); + SharedStrings.Add(shared.Key); } } else diff --git a/packages.config b/packages.config index 47b4718..146db44 100644 --- a/packages.config +++ b/packages.config @@ -1,6 +1,8 @@  - - + + + + \ No newline at end of file