Added support for SharedStrings and SSTR chunk type

BinaryFormat/BinaryReader.cs

@@ -10,10 +10,10 @@ namespace RobloxFiles.BinaryFormat
         public BinaryRobloxReader(Stream stream) : base(stream) { }
         private byte[] lastStringBuffer = new byte[0] { };
 
-        // Reads 'count * sizeof(T)' interleaved bytes and converts them
+        // Reads 'count * sizeof(T)' interleaved bytes and converts 
-        // into an array of T[count] where each value in the array has 
+        // them into an array of T[count] where each value in the 
-        // been transformed by the provided 'transform' function.
+        // array has been decoded by the provided 'decode' function.
-        public T[] ReadInterleaved<T>(int count, Func<byte[], int, T> transform) where T : struct
+        public T[] ReadInterleaved<T>(int count, Func<byte[], int, T> decode) where T : struct
         {
             int bufferSize = Marshal.SizeOf<T>();
             byte[] interleaved = ReadBytes(count * bufferSize);
@@ -32,21 +32,21 @@ namespace RobloxFiles.BinaryFormat
                 }
 
                 byte[] sequence = BitConverter.GetBytes(buffer);
-                values[i] = transform(sequence, 0);
+                values[i] = decode(sequence, 0);
             }
 
             return values;
         }
         
-        // Transforms an int from an interleaved buffer.
+        // Decodes an int from an interleaved buffer.
-        private int TransformInt(byte[] buffer, int startIndex)
+        private int DecodeInt(byte[] buffer, int startIndex)
         {
             int value = BitConverter.ToInt32(buffer, startIndex);
             return (value >> 1) ^ (-(value & 1));
         }
         
-        // Transforms a float from an interleaved buffer.
+        // Decodes a float from an interleaved buffer.
-        private float TransformFloat(byte[] buffer, int startIndex)
+        private float DecodeFloat(byte[] buffer, int startIndex)
         {
             uint u = BitConverter.ToUInt32(buffer, startIndex);
             uint i = (u >> 1) | (u << 31);
@@ -58,13 +58,19 @@ namespace RobloxFiles.BinaryFormat
         // Reads an interleaved buffer of integers.
         public int[] ReadInts(int count)
         {
-            return ReadInterleaved(count, TransformInt);
+            return ReadInterleaved(count, DecodeInt);
         }
         
         // Reads an interleaved buffer of floats.
         public float[] ReadFloats(int count)
         {
-            return ReadInterleaved(count, TransformFloat);
+            return ReadInterleaved(count, DecodeFloat);
+        }
+
+        // Reads an interleaved buffer of unsigned integers.
+        public uint[] ReadUInts(int count)
+        {
+            return ReadInterleaved(count, BitConverter.ToUInt32);
         }
         
         // Reads and accumulates an interleaved buffer of integers.

BinaryFormat/BinaryRobloxFile.cs

@@ -26,9 +26,11 @@ namespace RobloxFiles.BinaryFormat
         public override string ToString() => GetType().Name;
         
         public Instance[] Instances;
-        public META Metadata;
         public INST[] Types;
 
+        public Dictionary<string, string> Metadata;
+        public Dictionary<uint, string> SharedStrings;
+        
         public void ReadFile(byte[] contents)
         {
             using (MemoryStream file = new MemoryStream(contents))
@@ -75,12 +77,18 @@ namespace RobloxFiles.BinaryFormat
                                 hierarchy.Assemble(this);
                                 break;
                             case "META":
-                                Metadata = new META(chunk);
+                                META meta = new META(chunk);
+                                Metadata = meta.Data;
+                                break;
+                            case "SSTR":
+                                SSTR shared = new SSTR(chunk);
+                                SharedStrings = shared.Strings;
                                 break;
                             case "END\0":
                                 reading = false;
                                 break;
                             default:
+                                Console.WriteLine("Unhandled chunk type: {0}!", chunk.ChunkType);
                                 Chunks.Remove(chunk);
                                 break;
                         }

BinaryFormat/ChunkTypes/META.cs

@@ -5,20 +5,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
     public class META
     {
         public int NumEntries;
-        public Dictionary<string, string> Entries;
+        public Dictionary<string, string> Data = new Dictionary<string, string>();
 
         public META(BinaryRobloxChunk chunk)
         {
             using (BinaryRobloxReader reader = chunk.GetReader("META"))
             {
                 NumEntries = reader.ReadInt32();
-                Entries = new Dictionary<string, string>(NumEntries);
 
                 for (int i = 0; i < NumEntries; i++)
                 {
                     string key = reader.ReadString();
                     string value = reader.ReadString();
-                    Entries.Add(key, value);
+                    Data.Add(key, value);
                 }
             }
         }

BinaryFormat/ChunkTypes/PROP.cs

@@ -1,9 +1,10 @@
 using System;
+using System.IO;
 using System.Linq;
 
 using RobloxFiles.Enums;
 using RobloxFiles.DataTypes;
-using RobloxFiles.DataTypes.Utility;
+using RobloxFiles.Utility;
 
 namespace RobloxFiles.BinaryFormat.Chunks
 {
@@ -44,21 +45,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
             for (int i = 0; i < instCount; i++)
             {
                 int id = ids[i];
-                Instance instance = file.Instances[id];
+                Instance inst = file.Instances[id];
-
-                Property prop = new Property();
-                prop.Name = Name;
-                prop.Type = Type;
-                prop.Instance = instance;
 
+                Property prop = new Property(inst, this);
                 props[i] = prop;
-                instance.AddProperty(ref prop);
+
+                inst.AddProperty(ref prop);
             }
 
-            // Setup some short-hand functions for actions frequently used during the read procedure.
+            // Setup some short-hand functions for actions used during the read procedure.
             var readInts = new Func<int[]>(() => Reader.ReadInts(instCount));
             var readFloats = new Func<float[]>(() => Reader.ReadFloats(instCount));
             
+            
             var loadProperties = new Action<Func<int, object>>(read =>
             {
                 for (int i = 0; i < instCount; i++)
@@ -290,7 +289,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
                     // TODO: I want to map these values to actual Roblox enums, but I'll have to add an
                     //       interpreter for the JSON API Dump to do it properly.
 
-                    uint[] enums = Reader.ReadInterleaved(instCount, BitConverter.ToUInt32);
+                    uint[] enums = Reader.ReadUInts(instCount);
                     loadProperties(i => enums[i]);
 
                     break;
@@ -431,6 +430,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
 
                     loadProperties(i => int64s[i]);
                     break;
+                case PropertyType.SharedString:
+                    uint[] sharedKeys = Reader.ReadUInts(instCount);
+
+                    loadProperties(i =>
+                    {
+                        uint key = sharedKeys[i];
+                        return file.SharedStrings[key];
+                    });
+
+                    break;
+                default:
+                    Console.WriteLine("Unhandled property type: {0}!", Type);
+                    break;
             }
 
             Reader.Dispose();

BinaryFormat/ChunkTypes/SSTR.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+
+namespace RobloxFiles.BinaryFormat.Chunks
+{
+    public class SSTR
+    {
+        public int Version;
+        public int NumHashes;
+
+        public Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
+        public Dictionary<uint, string> Strings = new Dictionary<uint, string>();
+
+        public SSTR(BinaryRobloxChunk chunk)
+        {
+            using (BinaryRobloxReader reader = chunk.GetReader("SSTR"))
+            {
+                Version = reader.ReadInt32();
+                NumHashes = reader.ReadInt32();
+
+                for (uint id = 0; id < NumHashes; id++)
+                {
+                    byte[] md5 = reader.ReadBytes(16);
+
+                    int length = reader.ReadInt32();
+                    byte[] data = reader.ReadBytes(length);
+
+                    string key = Convert.ToBase64String(md5);
+                    string value = Convert.ToBase64String(data);
+
+                    Lookup.Add(key, id);
+                    Strings.Add(id, value);
+                }
+            }
+        }
+    }
+}

DataTypes/CFrame.cs

@@ -1,5 +1,5 @@
 using System;
-using RobloxFiles.DataTypes.Utility;
+using RobloxFiles.Utility;
 
 namespace RobloxFiles.DataTypes
 {

RobloxFileFormat.csproj

@@ -69,6 +69,7 @@
     <Compile Include="BinaryFormat\ChunkTypes\META.cs" />
     <Compile Include="BinaryFormat\ChunkTypes\PRNT.cs" />
     <Compile Include="BinaryFormat\ChunkTypes\PROP.cs" />
+    <Compile Include="BinaryFormat\ChunkTypes\SSTR.cs" />
     <Compile Include="Tree\Enums.cs" />
     <Compile Include="Tree\Property.cs" />
     <Compile Include="Tree\Instance.cs" />
@@ -99,6 +100,7 @@
     <Compile Include="Utility\MaterialInfo.cs" />
     <Compile Include="Utility\Quaternion.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="XmlFormat\PropertyTokens\SharedString.cs" />
     <Compile Include="XmlFormat\PropertyTokens\Vector3int16.cs" />
     <Compile Include="XmlFormat\XmlPropertyTokens.cs" />
     <Compile Include="XmlFormat\XmlDataReader.cs" />

Tree/Instance.cs

@@ -15,7 +15,7 @@ namespace RobloxFiles
         public readonly string ClassName;
 
         /// <summary>A list of properties that are defined under this Instance.</summary>
-        public List<Property> Properties = new List<Property>();
+        public Dictionary<string, Property> Properties = new Dictionary<string, Property>();
         
         private List<Instance> Children = new List<Instance>();
         private Instance rawParent;
@@ -36,14 +36,16 @@ namespace RobloxFiles
         /// <param name="name">The Name to use for this Instance.</param>
         public Instance(string className = "Instance", string name = "Instance")
         {
-            Property propName = new Property();
+            Property propName = new Property()
-            propName.Name = "Name";
+            {
-            propName.Type = PropertyType.String;
+                Type = PropertyType.String,
-            propName.Value = name;
+                Instance = this,
-            propName.Instance = this;
+                Name = "Name",
+                Value = name,
+            };
 
             ClassName = className;
-            Properties.Add(propName);
+            AddProperty(ref propName);
         }
 
         /// <summary>Returns true if this Instance is an ancestor to the provided Instance.</summary>
@@ -235,9 +237,8 @@ namespace RobloxFiles
         {
             Property property = null;
 
-            var query = Properties.Where((prop) => prop.Name.ToLower() == propertyName.ToLower());
+            if (Properties.ContainsKey(propertyName))
-            if (query.Count() > 0)
+                property = Properties[propertyName];
-                property = query.First();
 
             return (property != null ? property.Value : null);
         }
@@ -293,7 +294,7 @@ namespace RobloxFiles
         /// <param name="prop">A reference to the property that will be added.</param>
         public void AddProperty(ref Property prop)
         {
-            Properties.Add(prop);
+            Properties.Add(prop.Name, prop);
         }
 
         /// <summary>
@@ -319,18 +320,14 @@ namespace RobloxFiles
                     if (next == null)
                     {
                         // Check if there is any property with this name.
-                        var propQuery = result.Properties
+                        Property prop = null;
-                            .Where((prop) => name == prop.Name);
 
-                        if (propQuery.Count() > 0)
+                        if (result.Properties.ContainsKey(name))
-                        {
+                            prop = result.Properties[name];
-                            var prop = propQuery.First();
-                            return prop;
-                        }
                         else
-                        {
                             throw new Exception(name + " is not a valid member of " + result.Name);
-                        }
+
+                        return prop;
                     }
 
                     result = next;

Tree/Property.cs

@@ -1,4 +1,5 @@
 using System;
+using RobloxFiles.BinaryFormat.Chunks;
 
 namespace RobloxFiles
 {
@@ -31,7 +32,8 @@ namespace RobloxFiles
         Rect,
         PhysicalProperties,
         Color3uint8,
-        Int64
+        Int64,
+        SharedString
     }
 
     public class Property
@@ -67,6 +69,9 @@ namespace RobloxFiles
                         case PropertyType.Double:
                             RawBuffer = BitConverter.GetBytes((double)Value);
                             break;
+                        case PropertyType.SharedString:
+                            RawBuffer = Convert.FromBase64String((string)Value);
+                            break;
                     }
                 }
 
@@ -74,6 +79,21 @@ namespace RobloxFiles
             }
         }
 
+        public Property(string name = "", PropertyType type = PropertyType.Unknown, Instance instance = null)
+        {
+            Name = name;
+            Type = type;
+
+            Instance = instance;
+        }
+
+        public Property(Instance instance, PROP property)
+        {
+            Instance = instance;
+            Name = property.Name;
+            Type = property.Type;
+        }
+
         public string GetFullName()
         {
             string result = Name;

Utility/Quaternion.cs

@@ -1,6 +1,7 @@
 using System;
+using RobloxFiles.DataTypes;
 
-namespace RobloxFiles.DataTypes.Utility
+namespace RobloxFiles.Utility
 {
     /// <summary>
     /// Quaternion is a utility used by the CFrame DataType to handle rotation interpolation.

XmlFormat/PropertyTokens/Color3.cs

@@ -7,21 +7,16 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
     public class Color3Token : IXmlPropertyToken
     {
         public string Token => "Color3";
-        private string[] LegacyFields = new string[3] { "R", "G", "B" };
+        private string[] Fields = new string[3] { "R", "G", "B" };
 
         public bool ReadToken(Property prop, XmlNode token)
         {
-            var color3uint8 = XmlPropertyTokens.GetHandler<Color3uint8Token>();
+            bool success = true;
-            bool success = color3uint8.ReadToken(prop, token);
+            float[] fields = new float[Fields.Length];
-
-            if (!success)
-            {
-                // Try the legacy technique.
-                float[] fields = new float[LegacyFields.Length];
 
             for (int i = 0; i < fields.Length; i++)
             {
-                    string key = LegacyFields[i];
+                string key = Fields[i];
 
                 try
                 {
@@ -30,18 +25,25 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
                 }
                 catch
                 {
-                        return false;
+                    success = false;
+                    break;
                 }
             }
 
+            if (success)
+            {
                 float r = fields[0],
                       g = fields[1],
                       b = fields[2];
 
                 prop.Type = PropertyType.Color3;
                 prop.Value = new Color3(r, g, b);
-
+            }
-                success = true;
+            else
+            {
+                // Try falling back to the Color3uint8 technique...
+                var color3uint8 = XmlPropertyTokens.GetHandler<Color3uint8Token>();
+                success = color3uint8.ReadToken(prop, token);
             }
 
             return success;

XmlFormat/PropertyTokens/SharedString.cs

@@ -0,0 +1,19 @@
+using System.Text;
+using System.Xml;
+
+namespace RobloxFiles.XmlFormat.PropertyTokens
+{
+    public class SharedStringToken : IXmlPropertyToken
+    {
+        public string Token => "SharedString";
+
+        public bool ReadToken(Property prop, XmlNode token)
+        {
+            string contents = token.InnerText;
+            prop.Type = PropertyType.SharedString;
+            prop.Value = contents;
+
+            return true;
+        }
+    }
+}

XmlFormat/XmlDataReader.cs

@@ -1,15 +1,51 @@
 using System;
-using System.Collections.Generic;
 using System.Xml;
 
 namespace RobloxFiles.XmlFormat
 {
     public static class XmlDataReader
     {
+        private static Func<string, Exception> createErrorHandler(string label)
+        {
+            var errorHandler = new Func<string, Exception>((message) =>
+            {
+                string contents = $"XmlDataReader.{label}: {message}";
+                return new Exception(contents);
+            });
+
+            return errorHandler;
+        }
+
+        public static void ReadSharedStrings(XmlNode sharedStrings, XmlRobloxFile file)
+        {
+            var error = createErrorHandler("ReadSharedStrings");
+
+            if (sharedStrings.Name != "SharedStrings")
+                throw error("Provided XmlNode's class should be 'SharedStrings'!");
+
+            foreach (XmlNode sharedString in sharedStrings)
+            {
+                if (sharedString.Name == "SharedString")
+                {
+                    XmlNode md5Node = sharedString.Attributes.GetNamedItem("md5");
+
+                    if (md5Node == null)
+                        throw error("Got a SharedString without an 'md5' attribute!");
+
+                    string key = md5Node.InnerText;
+                    string value = sharedString.InnerText.Replace("\n", "");
+
+                    file.SharedStrings.Add(key, value);
+                }
+            }
+        }
+
         public static void ReadProperties(Instance instance, XmlNode propsNode)
         {
+            var error = createErrorHandler("ReadProperties");
+
             if (propsNode.Name != "Properties")
-                throw new Exception("XmlDataReader.ReadProperties: Provided XmlNode's class should be 'Properties'!");
+                throw error("Provided XmlNode's class should be 'Properties'!");
 
             foreach (XmlNode propNode in propsNode.ChildNodes)
             {
@@ -17,7 +53,7 @@ namespace RobloxFiles.XmlFormat
                 XmlNode propName = propNode.Attributes.GetNamedItem("name");
 
                 if (propName == null)
-                    throw new Exception("XmlDataReader.ReadProperties: Got a property node without a 'name' attribute!");
+                    throw error("Got a property node without a 'name' attribute!");
 
                 IXmlPropertyToken tokenHandler = XmlPropertyTokens.GetHandler(propType);
 
@@ -28,26 +64,28 @@ namespace RobloxFiles.XmlFormat
                     prop.Instance = instance;
 
                     if (!tokenHandler.ReadToken(prop, propNode))
-                        Console.WriteLine("XmlDataReader.ReadProperties:  Could not read property: " + prop.GetFullName() + '!');
+                        Console.WriteLine("Could not read property: " + prop.GetFullName() + '!');
 
                     instance.AddProperty(ref prop);
                 }
                 else
                 {
-                    Console.WriteLine("XmlDataReader.ReadProperties: No IXmlPropertyToken found for property type: " + propType + '!');
+                    Console.WriteLine("No IXmlPropertyToken found for property type: " + propType + '!');
                 }
             }
         }
 
         public static Instance ReadInstance(XmlNode instNode, XmlRobloxFile file = null)
         {
+            var error = createErrorHandler("ReadInstance");
+
             // Process the instance itself
             if (instNode.Name != "Item")
-                throw new Exception("XmlDataReader.ReadInstance: Provided XmlNode's name should be 'Item'!");
+                throw error("Provided XmlNode's name should be 'Item'!");
 
             XmlNode classToken = instNode.Attributes.GetNamedItem("class");
             if (classToken == null)
-                throw new Exception("XmlDataReader.ReadInstance: Got an Item without a defined 'class' attribute!");
+                throw error("Got an Item without a defined 'class' attribute!");
 
             Instance inst = new Instance(classToken.InnerText);
 
@@ -59,7 +97,7 @@ namespace RobloxFiles.XmlFormat
                 string refId = refToken.InnerText;
 
                 if (file.Instances.ContainsKey(refId))
-                    throw new Exception("XmlDataReader.ReadInstance: Got an Item with a duplicate 'referent' attribute!");
+                    throw error("Got an Item with a duplicate 'referent' attribute!");
 
                 file.Instances.Add(refId, inst);
             }

XmlFormat/XmlPropertyTokens.cs

@@ -36,21 +36,23 @@ namespace RobloxFiles.XmlFormat
         }
 
         public static bool ReadTokenGeneric<T>(Property prop, PropertyType propType, XmlNode token) where T : struct
+        {
+            try
             {
                 Type resultType = typeof(T);
                 TypeConverter converter = TypeDescriptor.GetConverter(resultType);
 
-            if (converter != null)
-            {
                 object result = converter.ConvertFromString(token.InnerText);
                 prop.Type = propType;
                 prop.Value = result;
 
                 return true;
             }
-
+            catch
+            {
                 return false;
             }
+        }
 
         public static IXmlPropertyToken GetHandler(string tokenName)
         {

XmlFormat/XmlRobloxFile.cs

@@ -16,6 +16,7 @@ namespace RobloxFiles.XmlFormat
         // Runtime Specific
         public readonly XmlDocument Root = new XmlDocument();
         public Dictionary<string, Instance> Instances = new Dictionary<string, Instance>();
+        public Dictionary<string, string> SharedStrings = new Dictionary<string, string>();
         
         public void ReadFile(byte[] buffer)
         {
@@ -50,16 +51,24 @@ namespace RobloxFiles.XmlFormat
                         Instance item = XmlDataReader.ReadInstance(child, this);
                         item.Parent = XmlContents;
                     }
+                    else if (child.Name == "SharedStrings")
+                    {
+                        XmlDataReader.ReadSharedStrings(child, this);
+                    }
                 }
 
-                // Resolve referent properties.
+                // Query the properties.
-                var refProps = Instances.Values
+                var props = Instances.Values
                     .SelectMany(inst => inst.Properties)
-                    .Where(prop => prop.Type == PropertyType.Ref);
+                    .Select(pair => pair.Value);
+
+                // Resolve referent properties.
+                var refProps = props.Where(prop => prop.Type == PropertyType.Ref);
                   
                 foreach (Property refProp in refProps)
                 {
                     string refId = refProp.Value as string;
+
                     if (Instances.ContainsKey(refId))
                     {
                         Instance refInst = Instances[refId];
@@ -67,13 +76,36 @@ namespace RobloxFiles.XmlFormat
                     }
                     else if (refId != "null")
                     {
-                        Console.WriteLine("XmlRobloxFile: Could not resolve reference for " + refProp.GetFullName());
+                        string name = refProp.GetFullName();
+                        Console.WriteLine("XmlRobloxFile: Could not resolve reference for {0}", name);
                     }
                 }
+
+                // Resolve shared strings.
+                var sharedProps = props.Where(prop => prop.Type == PropertyType.SharedString);
+
+                foreach (Property sharedProp in sharedProps)
+                {
+                    string md5 = sharedProp.Value as string;
+
+                    if (SharedStrings.ContainsKey(md5))
+                    {
+                        string value = SharedStrings[md5];
+                        sharedProp.Value = value;
+
+                        byte[] data = Convert.FromBase64String(value);
+                        sharedProp.SetRawBuffer(data);
+
+                        continue;
+                    }
+
+                    string name = sharedProp.GetFullName();
+                    Console.WriteLine("XmlRobloxFile: Could not resolve shared string for {0}", name);
+                }
             }
             else
             {
-                throw new Exception("XmlRobloxFile: No `roblox` tag found!");
+                throw new Exception("XmlRobloxFile: No 'roblox' tag found!");
             }
         }
     }