diff --git a/BinaryFormat/BinaryReader.cs b/BinaryFormat/BinaryReader.cs index 7d983d2..8e04d86 100644 --- a/BinaryFormat/BinaryReader.cs +++ b/BinaryFormat/BinaryReader.cs @@ -9,11 +9,11 @@ namespace RobloxFiles.BinaryFormat { public BinaryRobloxReader(Stream stream) : base(stream) { } private byte[] lastStringBuffer = new byte[0] { }; - - // Reads 'count * sizeof(T)' interleaved bytes and converts them - // into an array of T[count] where each value in the array has - // been transformed by the provided 'transform' function. - public T[] ReadInterleaved(int count, Func transform) where T : struct + + // Reads 'count * sizeof(T)' interleaved bytes and converts + // them into an array of T[count] where each value in the + // array has been decoded by the provided 'decode' function. + public T[] ReadInterleaved(int count, Func decode) where T : struct { int bufferSize = Marshal.SizeOf(); byte[] interleaved = ReadBytes(count * bufferSize); @@ -32,21 +32,21 @@ namespace RobloxFiles.BinaryFormat } byte[] sequence = BitConverter.GetBytes(buffer); - values[i] = transform(sequence, 0); + values[i] = decode(sequence, 0); } return values; } - // Transforms an int from an interleaved buffer. - private int TransformInt(byte[] buffer, int startIndex) + // Decodes an int from an interleaved buffer. + private int DecodeInt(byte[] buffer, int startIndex) { int value = BitConverter.ToInt32(buffer, startIndex); return (value >> 1) ^ (-(value & 1)); } - // Transforms a float from an interleaved buffer. - private float TransformFloat(byte[] buffer, int startIndex) + // Decodes a float from an interleaved buffer. + private float DecodeFloat(byte[] buffer, int startIndex) { uint u = BitConverter.ToUInt32(buffer, startIndex); uint i = (u >> 1) | (u << 31); @@ -58,13 +58,19 @@ namespace RobloxFiles.BinaryFormat // Reads an interleaved buffer of integers. public int[] ReadInts(int count) { - return ReadInterleaved(count, TransformInt); + return ReadInterleaved(count, DecodeInt); } // Reads an interleaved buffer of floats. public float[] ReadFloats(int count) { - return ReadInterleaved(count, TransformFloat); + return ReadInterleaved(count, DecodeFloat); + } + + // Reads an interleaved buffer of unsigned integers. + public uint[] ReadUInts(int count) + { + return ReadInterleaved(count, BitConverter.ToUInt32); } // Reads and accumulates an interleaved buffer of integers. diff --git a/BinaryFormat/BinaryRobloxFile.cs b/BinaryFormat/BinaryRobloxFile.cs index bc50153..bd81f3f 100644 --- a/BinaryFormat/BinaryRobloxFile.cs +++ b/BinaryFormat/BinaryRobloxFile.cs @@ -26,8 +26,10 @@ namespace RobloxFiles.BinaryFormat public override string ToString() => GetType().Name; public Instance[] Instances; - public META Metadata; public INST[] Types; + + public Dictionary Metadata; + public Dictionary SharedStrings; public void ReadFile(byte[] contents) { @@ -75,12 +77,18 @@ namespace RobloxFiles.BinaryFormat hierarchy.Assemble(this); break; case "META": - Metadata = new META(chunk); + META meta = new META(chunk); + Metadata = meta.Data; + break; + case "SSTR": + SSTR shared = new SSTR(chunk); + SharedStrings = shared.Strings; break; case "END\0": reading = false; break; default: + Console.WriteLine("Unhandled chunk type: {0}!", chunk.ChunkType); Chunks.Remove(chunk); break; } diff --git a/BinaryFormat/ChunkTypes/META.cs b/BinaryFormat/ChunkTypes/META.cs index 029ce0c..bff903a 100644 --- a/BinaryFormat/ChunkTypes/META.cs +++ b/BinaryFormat/ChunkTypes/META.cs @@ -5,20 +5,19 @@ namespace RobloxFiles.BinaryFormat.Chunks public class META { public int NumEntries; - public Dictionary Entries; + public Dictionary Data = new Dictionary(); public META(BinaryRobloxChunk chunk) { using (BinaryRobloxReader reader = chunk.GetReader("META")) { NumEntries = reader.ReadInt32(); - Entries = new Dictionary(NumEntries); for (int i = 0; i < NumEntries; i++) { string key = reader.ReadString(); string value = reader.ReadString(); - Entries.Add(key, value); + Data.Add(key, value); } } } diff --git a/BinaryFormat/ChunkTypes/PROP.cs b/BinaryFormat/ChunkTypes/PROP.cs index b6bc8c9..069b286 100644 --- a/BinaryFormat/ChunkTypes/PROP.cs +++ b/BinaryFormat/ChunkTypes/PROP.cs @@ -1,9 +1,10 @@ using System; +using System.IO; using System.Linq; using RobloxFiles.Enums; using RobloxFiles.DataTypes; -using RobloxFiles.DataTypes.Utility; +using RobloxFiles.Utility; namespace RobloxFiles.BinaryFormat.Chunks { @@ -44,21 +45,19 @@ namespace RobloxFiles.BinaryFormat.Chunks for (int i = 0; i < instCount; i++) { int id = ids[i]; - Instance instance = file.Instances[id]; - - Property prop = new Property(); - prop.Name = Name; - prop.Type = Type; - prop.Instance = instance; + Instance inst = file.Instances[id]; + Property prop = new Property(inst, this); props[i] = prop; - instance.AddProperty(ref prop); + + inst.AddProperty(ref prop); } - // Setup some short-hand functions for actions frequently used during the read procedure. + // Setup some short-hand functions for actions used during the read procedure. var readInts = new Func(() => Reader.ReadInts(instCount)); var readFloats = new Func(() => Reader.ReadFloats(instCount)); + var loadProperties = new Action>(read => { for (int i = 0; i < instCount; i++) @@ -290,7 +289,7 @@ namespace RobloxFiles.BinaryFormat.Chunks // TODO: I want to map these values to actual Roblox enums, but I'll have to add an // interpreter for the JSON API Dump to do it properly. - uint[] enums = Reader.ReadInterleaved(instCount, BitConverter.ToUInt32); + uint[] enums = Reader.ReadUInts(instCount); loadProperties(i => enums[i]); break; @@ -431,6 +430,19 @@ namespace RobloxFiles.BinaryFormat.Chunks loadProperties(i => int64s[i]); break; + case PropertyType.SharedString: + uint[] sharedKeys = Reader.ReadUInts(instCount); + + loadProperties(i => + { + uint key = sharedKeys[i]; + return file.SharedStrings[key]; + }); + + break; + default: + Console.WriteLine("Unhandled property type: {0}!", Type); + break; } Reader.Dispose(); diff --git a/BinaryFormat/ChunkTypes/SSTR.cs b/BinaryFormat/ChunkTypes/SSTR.cs new file mode 100644 index 0000000..7a897c0 --- /dev/null +++ b/BinaryFormat/ChunkTypes/SSTR.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; + +namespace RobloxFiles.BinaryFormat.Chunks +{ + public class SSTR + { + public int Version; + public int NumHashes; + + public Dictionary Lookup = new Dictionary(); + public Dictionary Strings = new Dictionary(); + + public SSTR(BinaryRobloxChunk chunk) + { + using (BinaryRobloxReader reader = chunk.GetReader("SSTR")) + { + Version = reader.ReadInt32(); + NumHashes = reader.ReadInt32(); + + for (uint id = 0; id < NumHashes; id++) + { + byte[] md5 = reader.ReadBytes(16); + + int length = reader.ReadInt32(); + byte[] data = reader.ReadBytes(length); + + string key = Convert.ToBase64String(md5); + string value = Convert.ToBase64String(data); + + Lookup.Add(key, id); + Strings.Add(id, value); + } + } + } + } +} diff --git a/DataTypes/CFrame.cs b/DataTypes/CFrame.cs index a85192e..8545d9c 100644 --- a/DataTypes/CFrame.cs +++ b/DataTypes/CFrame.cs @@ -1,5 +1,5 @@ using System; -using RobloxFiles.DataTypes.Utility; +using RobloxFiles.Utility; namespace RobloxFiles.DataTypes { diff --git a/RobloxFileFormat.csproj b/RobloxFileFormat.csproj index c706faf..edeab34 100644 --- a/RobloxFileFormat.csproj +++ b/RobloxFileFormat.csproj @@ -69,6 +69,7 @@ + @@ -99,6 +100,7 @@ + diff --git a/Tree/Instance.cs b/Tree/Instance.cs index 4160707..629bc8c 100644 --- a/Tree/Instance.cs +++ b/Tree/Instance.cs @@ -15,7 +15,7 @@ namespace RobloxFiles public readonly string ClassName; /// A list of properties that are defined under this Instance. - public List Properties = new List(); + public Dictionary Properties = new Dictionary(); private List Children = new List(); private Instance rawParent; @@ -36,14 +36,16 @@ namespace RobloxFiles /// The Name to use for this Instance. public Instance(string className = "Instance", string name = "Instance") { - Property propName = new Property(); - propName.Name = "Name"; - propName.Type = PropertyType.String; - propName.Value = name; - propName.Instance = this; + Property propName = new Property() + { + Type = PropertyType.String, + Instance = this, + Name = "Name", + Value = name, + }; ClassName = className; - Properties.Add(propName); + AddProperty(ref propName); } /// Returns true if this Instance is an ancestor to the provided Instance. @@ -235,9 +237,8 @@ namespace RobloxFiles { Property property = null; - var query = Properties.Where((prop) => prop.Name.ToLower() == propertyName.ToLower()); - if (query.Count() > 0) - property = query.First(); + if (Properties.ContainsKey(propertyName)) + property = Properties[propertyName]; return (property != null ? property.Value : null); } @@ -293,7 +294,7 @@ namespace RobloxFiles /// A reference to the property that will be added. public void AddProperty(ref Property prop) { - Properties.Add(prop); + Properties.Add(prop.Name, prop); } /// @@ -319,18 +320,14 @@ namespace RobloxFiles if (next == null) { // Check if there is any property with this name. - var propQuery = result.Properties - .Where((prop) => name == prop.Name); + Property prop = null; - if (propQuery.Count() > 0) - { - var prop = propQuery.First(); - return prop; - } + if (result.Properties.ContainsKey(name)) + prop = result.Properties[name]; else - { throw new Exception(name + " is not a valid member of " + result.Name); - } + + return prop; } result = next; diff --git a/Tree/Property.cs b/Tree/Property.cs index 5e261ef..dd5fdbf 100644 --- a/Tree/Property.cs +++ b/Tree/Property.cs @@ -1,4 +1,5 @@ using System; +using RobloxFiles.BinaryFormat.Chunks; namespace RobloxFiles { @@ -31,7 +32,8 @@ namespace RobloxFiles Rect, PhysicalProperties, Color3uint8, - Int64 + Int64, + SharedString } public class Property @@ -67,6 +69,9 @@ namespace RobloxFiles case PropertyType.Double: RawBuffer = BitConverter.GetBytes((double)Value); break; + case PropertyType.SharedString: + RawBuffer = Convert.FromBase64String((string)Value); + break; } } @@ -74,6 +79,21 @@ namespace RobloxFiles } } + public Property(string name = "", PropertyType type = PropertyType.Unknown, Instance instance = null) + { + Name = name; + Type = type; + + Instance = instance; + } + + public Property(Instance instance, PROP property) + { + Instance = instance; + Name = property.Name; + Type = property.Type; + } + public string GetFullName() { string result = Name; diff --git a/Utility/Quaternion.cs b/Utility/Quaternion.cs index aead226..608027c 100644 --- a/Utility/Quaternion.cs +++ b/Utility/Quaternion.cs @@ -1,6 +1,7 @@ using System; +using RobloxFiles.DataTypes; -namespace RobloxFiles.DataTypes.Utility +namespace RobloxFiles.Utility { /// /// Quaternion is a utility used by the CFrame DataType to handle rotation interpolation. diff --git a/XmlFormat/PropertyTokens/Color3.cs b/XmlFormat/PropertyTokens/Color3.cs index 8ba18a6..4ab89d3 100644 --- a/XmlFormat/PropertyTokens/Color3.cs +++ b/XmlFormat/PropertyTokens/Color3.cs @@ -7,43 +7,45 @@ namespace RobloxFiles.XmlFormat.PropertyTokens public class Color3Token : IXmlPropertyToken { public string Token => "Color3"; - private string[] LegacyFields = new string[3] { "R", "G", "B" }; + private string[] Fields = new string[3] { "R", "G", "B" }; public bool ReadToken(Property prop, XmlNode token) { - var color3uint8 = XmlPropertyTokens.GetHandler(); - bool success = color3uint8.ReadToken(prop, token); + bool success = true; + float[] fields = new float[Fields.Length]; - if (!success) + for (int i = 0; i < fields.Length; i++) { - // Try the legacy technique. - float[] fields = new float[LegacyFields.Length]; + string key = Fields[i]; - for (int i = 0; i < fields.Length; i++) + try { - string key = LegacyFields[i]; - - try - { - var coord = token[key]; - fields[i] = XmlPropertyTokens.ParseFloat(coord.InnerText); - } - catch - { - return false; - } + var coord = token[key]; + fields[i] = XmlPropertyTokens.ParseFloat(coord.InnerText); } + catch + { + success = false; + break; + } + } + if (success) + { float r = fields[0], g = fields[1], b = fields[2]; prop.Type = PropertyType.Color3; prop.Value = new Color3(r, g, b); - - success = true; } - + else + { + // Try falling back to the Color3uint8 technique... + var color3uint8 = XmlPropertyTokens.GetHandler(); + success = color3uint8.ReadToken(prop, token); + } + return success; } } diff --git a/XmlFormat/PropertyTokens/SharedString.cs b/XmlFormat/PropertyTokens/SharedString.cs new file mode 100644 index 0000000..6cb0140 --- /dev/null +++ b/XmlFormat/PropertyTokens/SharedString.cs @@ -0,0 +1,19 @@ +using System.Text; +using System.Xml; + +namespace RobloxFiles.XmlFormat.PropertyTokens +{ + public class SharedStringToken : IXmlPropertyToken + { + public string Token => "SharedString"; + + public bool ReadToken(Property prop, XmlNode token) + { + string contents = token.InnerText; + prop.Type = PropertyType.SharedString; + prop.Value = contents; + + return true; + } + } +} diff --git a/XmlFormat/XmlDataReader.cs b/XmlFormat/XmlDataReader.cs index 52c1b9b..da4f330 100644 --- a/XmlFormat/XmlDataReader.cs +++ b/XmlFormat/XmlDataReader.cs @@ -1,15 +1,51 @@ using System; -using System.Collections.Generic; using System.Xml; namespace RobloxFiles.XmlFormat { public static class XmlDataReader { + private static Func createErrorHandler(string label) + { + var errorHandler = new Func((message) => + { + string contents = $"XmlDataReader.{label}: {message}"; + return new Exception(contents); + }); + + return errorHandler; + } + + public static void ReadSharedStrings(XmlNode sharedStrings, XmlRobloxFile file) + { + var error = createErrorHandler("ReadSharedStrings"); + + if (sharedStrings.Name != "SharedStrings") + throw error("Provided XmlNode's class should be 'SharedStrings'!"); + + foreach (XmlNode sharedString in sharedStrings) + { + if (sharedString.Name == "SharedString") + { + XmlNode md5Node = sharedString.Attributes.GetNamedItem("md5"); + + if (md5Node == null) + throw error("Got a SharedString without an 'md5' attribute!"); + + string key = md5Node.InnerText; + string value = sharedString.InnerText.Replace("\n", ""); + + file.SharedStrings.Add(key, value); + } + } + } + public static void ReadProperties(Instance instance, XmlNode propsNode) { + var error = createErrorHandler("ReadProperties"); + if (propsNode.Name != "Properties") - throw new Exception("XmlDataReader.ReadProperties: Provided XmlNode's class should be 'Properties'!"); + throw error("Provided XmlNode's class should be 'Properties'!"); foreach (XmlNode propNode in propsNode.ChildNodes) { @@ -17,7 +53,7 @@ namespace RobloxFiles.XmlFormat XmlNode propName = propNode.Attributes.GetNamedItem("name"); if (propName == null) - throw new Exception("XmlDataReader.ReadProperties: Got a property node without a 'name' attribute!"); + throw error("Got a property node without a 'name' attribute!"); IXmlPropertyToken tokenHandler = XmlPropertyTokens.GetHandler(propType); @@ -28,26 +64,28 @@ namespace RobloxFiles.XmlFormat prop.Instance = instance; if (!tokenHandler.ReadToken(prop, propNode)) - Console.WriteLine("XmlDataReader.ReadProperties: Could not read property: " + prop.GetFullName() + '!'); + Console.WriteLine("Could not read property: " + prop.GetFullName() + '!'); instance.AddProperty(ref prop); } else { - Console.WriteLine("XmlDataReader.ReadProperties: No IXmlPropertyToken found for property type: " + propType + '!'); + Console.WriteLine("No IXmlPropertyToken found for property type: " + propType + '!'); } } } public static Instance ReadInstance(XmlNode instNode, XmlRobloxFile file = null) { + var error = createErrorHandler("ReadInstance"); + // Process the instance itself if (instNode.Name != "Item") - throw new Exception("XmlDataReader.ReadInstance: Provided XmlNode's name should be 'Item'!"); + throw error("Provided XmlNode's name should be 'Item'!"); XmlNode classToken = instNode.Attributes.GetNamedItem("class"); if (classToken == null) - throw new Exception("XmlDataReader.ReadInstance: Got an Item without a defined 'class' attribute!"); + throw error("Got an Item without a defined 'class' attribute!"); Instance inst = new Instance(classToken.InnerText); @@ -59,7 +97,7 @@ namespace RobloxFiles.XmlFormat string refId = refToken.InnerText; if (file.Instances.ContainsKey(refId)) - throw new Exception("XmlDataReader.ReadInstance: Got an Item with a duplicate 'referent' attribute!"); + throw error("Got an Item with a duplicate 'referent' attribute!"); file.Instances.Add(refId, inst); } diff --git a/XmlFormat/XmlPropertyTokens.cs b/XmlFormat/XmlPropertyTokens.cs index 1526752..570b266 100644 --- a/XmlFormat/XmlPropertyTokens.cs +++ b/XmlFormat/XmlPropertyTokens.cs @@ -37,19 +37,21 @@ namespace RobloxFiles.XmlFormat public static bool ReadTokenGeneric(Property prop, PropertyType propType, XmlNode token) where T : struct { - Type resultType = typeof(T); - TypeConverter converter = TypeDescriptor.GetConverter(resultType); - - if (converter != null) + try { + Type resultType = typeof(T); + TypeConverter converter = TypeDescriptor.GetConverter(resultType); + object result = converter.ConvertFromString(token.InnerText); prop.Type = propType; prop.Value = result; return true; } - - return false; + catch + { + return false; + } } public static IXmlPropertyToken GetHandler(string tokenName) diff --git a/XmlFormat/XmlRobloxFile.cs b/XmlFormat/XmlRobloxFile.cs index 1f34db3..0efe44d 100644 --- a/XmlFormat/XmlRobloxFile.cs +++ b/XmlFormat/XmlRobloxFile.cs @@ -16,6 +16,7 @@ namespace RobloxFiles.XmlFormat // Runtime Specific public readonly XmlDocument Root = new XmlDocument(); public Dictionary Instances = new Dictionary(); + public Dictionary SharedStrings = new Dictionary(); public void ReadFile(byte[] buffer) { @@ -50,16 +51,24 @@ namespace RobloxFiles.XmlFormat Instance item = XmlDataReader.ReadInstance(child, this); item.Parent = XmlContents; } + else if (child.Name == "SharedStrings") + { + XmlDataReader.ReadSharedStrings(child, this); + } } - // Resolve referent properties. - var refProps = Instances.Values + // Query the properties. + var props = Instances.Values .SelectMany(inst => inst.Properties) - .Where(prop => prop.Type == PropertyType.Ref); - + .Select(pair => pair.Value); + + // Resolve referent properties. + var refProps = props.Where(prop => prop.Type == PropertyType.Ref); + foreach (Property refProp in refProps) { string refId = refProp.Value as string; + if (Instances.ContainsKey(refId)) { Instance refInst = Instances[refId]; @@ -67,13 +76,36 @@ namespace RobloxFiles.XmlFormat } else if (refId != "null") { - Console.WriteLine("XmlRobloxFile: Could not resolve reference for " + refProp.GetFullName()); + string name = refProp.GetFullName(); + Console.WriteLine("XmlRobloxFile: Could not resolve reference for {0}", name); } } + + // Resolve shared strings. + var sharedProps = props.Where(prop => prop.Type == PropertyType.SharedString); + + foreach (Property sharedProp in sharedProps) + { + string md5 = sharedProp.Value as string; + + if (SharedStrings.ContainsKey(md5)) + { + string value = SharedStrings[md5]; + sharedProp.Value = value; + + byte[] data = Convert.FromBase64String(value); + sharedProp.SetRawBuffer(data); + + continue; + } + + string name = sharedProp.GetFullName(); + Console.WriteLine("XmlRobloxFile: Could not resolve shared string for {0}", name); + } } else { - throw new Exception("XmlRobloxFile: No `roblox` tag found!"); + throw new Exception("XmlRobloxFile: No 'roblox' tag found!"); } } }