Source2Roblox/Roblox-File-Format
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added support for SharedStrings and SSTR chunk type

Browse Source
This commit is contained in:
CloneTrooper1019 2019-05-17 01:14:04 -05:00
parent 32e80bdd9a
commit 45a84e34d0
15 changed files with 266 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
BinaryFormat/BinaryReader.cs
View File

@ -9,11 +9,11 @@ namespace RobloxFiles.BinaryFormat
{
public BinaryRobloxReader(Stream stream) : base(stream) { }
private byte[] lastStringBuffer = new byte[0] { };
// Reads 'count * sizeof(T)' interleaved bytes and converts them
// into an array of T[count] where each value in the array has
// been transformed by the provided 'transform' function.
public T[] ReadInterleaved<T>(int count, Func<byte[], int, T> transform) where T : struct
// Reads 'count * sizeof(T)' interleaved bytes and converts
// them into an array of T[count] where each value in the
// array has been decoded by the provided 'decode' function.
public T[] ReadInterleaved<T>(int count, Func<byte[], int, T> decode) where T : struct
{
int bufferSize = Marshal.SizeOf<T>();
byte[] interleaved = ReadBytes(count * bufferSize);
@ -32,21 +32,21 @@ namespace RobloxFiles.BinaryFormat
}
byte[] sequence = BitConverter.GetBytes(buffer);
values[i] = transform(sequence, 0);
values[i] = decode(sequence, 0);
}
return values;
}
// Transforms an int from an interleaved buffer.
private int TransformInt(byte[] buffer, int startIndex)
// Decodes an int from an interleaved buffer.
private int DecodeInt(byte[] buffer, int startIndex)
{
int value = BitConverter.ToInt32(buffer, startIndex);
return (value >> 1) ^ (-(value & 1));
}
// Transforms a float from an interleaved buffer.
private float TransformFloat(byte[] buffer, int startIndex)
// Decodes a float from an interleaved buffer.
private float DecodeFloat(byte[] buffer, int startIndex)
{
uint u = BitConverter.ToUInt32(buffer, startIndex);
uint i = (u >> 1) | (u << 31);
@ -58,13 +58,19 @@ namespace RobloxFiles.BinaryFormat
// Reads an interleaved buffer of integers.
public int[] ReadInts(int count)
{
return ReadInterleaved(count, TransformInt);
return ReadInterleaved(count, DecodeInt);
}
// Reads an interleaved buffer of floats.
public float[] ReadFloats(int count)
{
return ReadInterleaved(count, TransformFloat);
return ReadInterleaved(count, DecodeFloat);
}
// Reads an interleaved buffer of unsigned integers.
public uint[] ReadUInts(int count)
{
return ReadInterleaved(count, BitConverter.ToUInt32);
}
// Reads and accumulates an interleaved buffer of integers.

12
BinaryFormat/BinaryRobloxFile.cs
View File

@ -26,8 +26,10 @@ namespace RobloxFiles.BinaryFormat
public override string ToString() => GetType().Name;
public Instance[] Instances;
public META Metadata;
public INST[] Types;
public Dictionary<string, string> Metadata;
public Dictionary<uint, string> SharedStrings;
public void ReadFile(byte[] contents)
{
@ -75,12 +77,18 @@ namespace RobloxFiles.BinaryFormat
hierarchy.Assemble(this);
break;
case "META":
Metadata = new META(chunk);
META meta = new META(chunk);
Metadata = meta.Data;
break;
case "SSTR":
SSTR shared = new SSTR(chunk);
SharedStrings = shared.Strings;
break;
case "END\0":
reading = false;
break;
default:
Console.WriteLine("Unhandled chunk type: {0}!", chunk.ChunkType);
Chunks.Remove(chunk);
break;
}

5
BinaryFormat/ChunkTypes/META.cs
View File

@ -5,20 +5,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
public class META
{
public int NumEntries;
public Dictionary<string, string> Entries;
public Dictionary<string, string> Data = new Dictionary<string, string>();
public META(BinaryRobloxChunk chunk)
{
using (BinaryRobloxReader reader = chunk.GetReader("META"))
{
NumEntries = reader.ReadInt32();
Entries = new Dictionary<string, string>(NumEntries);
for (int i = 0; i < NumEntries; i++)
{
string key = reader.ReadString();
string value = reader.ReadString();
Entries.Add(key, value);
Data.Add(key, value);
}
}
}

32
BinaryFormat/ChunkTypes/PROP.cs
View File

@ -1,9 +1,10 @@
using System;
using System.IO;
using System.Linq;
using RobloxFiles.Enums;
using RobloxFiles.DataTypes;
using RobloxFiles.DataTypes.Utility;
using RobloxFiles.Utility;
namespace RobloxFiles.BinaryFormat.Chunks
{
@ -44,21 +45,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
for (int i = 0; i < instCount; i++)
{
int id = ids[i];
Instance instance = file.Instances[id];
Property prop = new Property();
prop.Name = Name;
prop.Type = Type;
prop.Instance = instance;
Instance inst = file.Instances[id];
Property prop = new Property(inst, this);
props[i] = prop;
instance.AddProperty(ref prop);
inst.AddProperty(ref prop);
}
// Setup some short-hand functions for actions frequently used during the read procedure.
// Setup some short-hand functions for actions used during the read procedure.
var readInts = new Func<int[]>(() => Reader.ReadInts(instCount));
var readFloats = new Func<float[]>(() => Reader.ReadFloats(instCount));
var loadProperties = new Action<Func<int, object>>(read =>
{
for (int i = 0; i < instCount; i++)
@ -290,7 +289,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
// TODO: I want to map these values to actual Roblox enums, but I'll have to add an
// interpreter for the JSON API Dump to do it properly.
uint[] enums = Reader.ReadInterleaved(instCount, BitConverter.ToUInt32);
uint[] enums = Reader.ReadUInts(instCount);
loadProperties(i => enums[i]);
break;
@ -431,6 +430,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
loadProperties(i => int64s[i]);
break;
case PropertyType.SharedString:
uint[] sharedKeys = Reader.ReadUInts(instCount);
loadProperties(i =>
{
uint key = sharedKeys[i];
return file.SharedStrings[key];
});
break;
default:
Console.WriteLine("Unhandled property type: {0}!", Type);
break;
}
Reader.Dispose();

37
BinaryFormat/ChunkTypes/SSTR.cs Normal file
View File

@ -0,0 +1,37 @@
using System;
using System.Collections.Generic;
namespace RobloxFiles.BinaryFormat.Chunks
{
public class SSTR
{
public int Version;
public int NumHashes;
public Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
public Dictionary<uint, string> Strings = new Dictionary<uint, string>();
public SSTR(BinaryRobloxChunk chunk)
{
using (BinaryRobloxReader reader = chunk.GetReader("SSTR"))
{
Version = reader.ReadInt32();
NumHashes = reader.ReadInt32();
for (uint id = 0; id < NumHashes; id++)
{
byte[] md5 = reader.ReadBytes(16);
int length = reader.ReadInt32();
byte[] data = reader.ReadBytes(length);
string key = Convert.ToBase64String(md5);
string value = Convert.ToBase64String(data);
Lookup.Add(key, id);
Strings.Add(id, value);
}
}
}
}
}

2
DataTypes/CFrame.cs
View File

@ -1,5 +1,5 @@
using System;
using RobloxFiles.DataTypes.Utility;
using RobloxFiles.Utility;
namespace RobloxFiles.DataTypes
{

2
RobloxFileFormat.csproj
View File

@ -69,6 +69,7 @@
<Compile Include="BinaryFormat\ChunkTypes\META.cs" />
<Compile Include="BinaryFormat\ChunkTypes\PRNT.cs" />
<Compile Include="BinaryFormat\ChunkTypes\PROP.cs" />
<Compile Include="BinaryFormat\ChunkTypes\SSTR.cs" />
<Compile Include="Tree\Enums.cs" />
<Compile Include="Tree\Property.cs" />
<Compile Include="Tree\Instance.cs" />
@ -99,6 +100,7 @@
<Compile Include="Utility\MaterialInfo.cs" />
<Compile Include="Utility\Quaternion.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="XmlFormat\PropertyTokens\SharedString.cs" />
<Compile Include="XmlFormat\PropertyTokens\Vector3int16.cs" />
<Compile Include="XmlFormat\XmlPropertyTokens.cs" />
<Compile Include="XmlFormat\XmlDataReader.cs" />

37
Tree/Instance.cs
View File

@ -15,7 +15,7 @@ namespace RobloxFiles
public readonly string ClassName;
/// <summary>A list of properties that are defined under this Instance.</summary>
public List<Property> Properties = new List<Property>();
public Dictionary<string, Property> Properties = new Dictionary<string, Property>();
private List<Instance> Children = new List<Instance>();
private Instance rawParent;
@ -36,14 +36,16 @@ namespace RobloxFiles
/// <param name="name">The Name to use for this Instance.</param>
public Instance(string className = "Instance", string name = "Instance")
{
Property propName = new Property();
propName.Name = "Name";
propName.Type = PropertyType.String;
propName.Value = name;
propName.Instance = this;
Property propName = new Property()
{
Type = PropertyType.String,
Instance = this,
Name = "Name",
Value = name,
};
ClassName = className;
Properties.Add(propName);
AddProperty(ref propName);
}
/// <summary>Returns true if this Instance is an ancestor to the provided Instance.</summary>
@ -235,9 +237,8 @@ namespace RobloxFiles
{
Property property = null;
var query = Properties.Where((prop) => prop.Name.ToLower() == propertyName.ToLower());
if (query.Count() > 0)
property = query.First();
if (Properties.ContainsKey(propertyName))
property = Properties[propertyName];
return (property != null ? property.Value : null);
}
@ -293,7 +294,7 @@ namespace RobloxFiles
/// <param name="prop">A reference to the property that will be added.</param>
public void AddProperty(ref Property prop)
{
Properties.Add(prop);
Properties.Add(prop.Name, prop);
}
/// <summary>
@ -319,18 +320,14 @@ namespace RobloxFiles
if (next == null)
{
// Check if there is any property with this name.
var propQuery = result.Properties
.Where((prop) => name == prop.Name);
Property prop = null;
if (propQuery.Count() > 0)
{
var prop = propQuery.First();
return prop;
}
if (result.Properties.ContainsKey(name))
prop = result.Properties[name];
else
{
throw new Exception(name + " is not a valid member of " + result.Name);
}
return prop;
}
result = next;

22
Tree/Property.cs
View File

@ -1,4 +1,5 @@
using System;
using RobloxFiles.BinaryFormat.Chunks;
namespace RobloxFiles
{
@ -31,7 +32,8 @@ namespace RobloxFiles
Rect,
PhysicalProperties,
Color3uint8,
Int64
Int64,
SharedString
}
public class Property
@ -67,6 +69,9 @@ namespace RobloxFiles
case PropertyType.Double:
RawBuffer = BitConverter.GetBytes((double)Value);
break;
case PropertyType.SharedString:
RawBuffer = Convert.FromBase64String((string)Value);
break;
}
}
@ -74,6 +79,21 @@ namespace RobloxFiles
}
}
public Property(string name = "", PropertyType type = PropertyType.Unknown, Instance instance = null)
{
Name = name;
Type = type;
Instance = instance;
}
public Property(Instance instance, PROP property)
{
Instance = instance;
Name = property.Name;
Type = property.Type;
}
public string GetFullName()
{
string result = Name;

3
Utility/Quaternion.cs
View File

@ -1,6 +1,7 @@
using System;
using RobloxFiles.DataTypes;
namespace RobloxFiles.DataTypes.Utility
namespace RobloxFiles.Utility
{
/// <summary>
/// Quaternion is a utility used by the CFrame DataType to handle rotation interpolation.

44
XmlFormat/PropertyTokens/Color3.cs
View File

@ -7,43 +7,45 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
public class Color3Token : IXmlPropertyToken
{
public string Token => "Color3";
private string[] LegacyFields = new string[3] { "R", "G", "B" };
private string[] Fields = new string[3] { "R", "G", "B" };
public bool ReadToken(Property prop, XmlNode token)
{
var color3uint8 = XmlPropertyTokens.GetHandler<Color3uint8Token>();
bool success = color3uint8.ReadToken(prop, token);
bool success = true;
float[] fields = new float[Fields.Length];
if (!success)
for (int i = 0; i < fields.Length; i++)
{
// Try the legacy technique.
float[] fields = new float[LegacyFields.Length];
string key = Fields[i];
for (int i = 0; i < fields.Length; i++)
try
{
string key = LegacyFields[i];
try
{
var coord = token[key];
fields[i] = XmlPropertyTokens.ParseFloat(coord.InnerText);
}
catch
{
return false;
}
var coord = token[key];
fields[i] = XmlPropertyTokens.ParseFloat(coord.InnerText);
}
catch
{
success = false;
break;
}
}
if (success)
{
float r = fields[0],
g = fields[1],
b = fields[2];
prop.Type = PropertyType.Color3;
prop.Value = new Color3(r, g, b);
success = true;
}
else
{
// Try falling back to the Color3uint8 technique...
var color3uint8 = XmlPropertyTokens.GetHandler<Color3uint8Token>();
success = color3uint8.ReadToken(prop, token);
}
return success;
}
}

19
XmlFormat/PropertyTokens/SharedString.cs Normal file
View File

@ -0,0 +1,19 @@
using System.Text;
using System.Xml;
namespace RobloxFiles.XmlFormat.PropertyTokens
{
public class SharedStringToken : IXmlPropertyToken
{
public string Token => "SharedString";
public bool ReadToken(Property prop, XmlNode token)
{
string contents = token.InnerText;
prop.Type = PropertyType.SharedString;
prop.Value = contents;
return true;
}
}
}

54
XmlFormat/XmlDataReader.cs
View File

@ -1,15 +1,51 @@
using System;
using System.Collections.Generic;
using System.Xml;
namespace RobloxFiles.XmlFormat
{
public static class XmlDataReader
{
private static Func<string, Exception> createErrorHandler(string label)
{
var errorHandler = new Func<string, Exception>((message) =>
{
string contents = $"XmlDataReader.{label}: {message}";
return new Exception(contents);
});
return errorHandler;
}
public static void ReadSharedStrings(XmlNode sharedStrings, XmlRobloxFile file)
{
var error = createErrorHandler("ReadSharedStrings");
if (sharedStrings.Name != "SharedStrings")
throw error("Provided XmlNode's class should be 'SharedStrings'!");
foreach (XmlNode sharedString in sharedStrings)
{
if (sharedString.Name == "SharedString")
{
XmlNode md5Node = sharedString.Attributes.GetNamedItem("md5");
if (md5Node == null)
throw error("Got a SharedString without an 'md5' attribute!");
string key = md5Node.InnerText;
string value = sharedString.InnerText.Replace("\n", "");
file.SharedStrings.Add(key, value);
}
}
}
public static void ReadProperties(Instance instance, XmlNode propsNode)
{
var error = createErrorHandler("ReadProperties");
if (propsNode.Name != "Properties")
throw new Exception("XmlDataReader.ReadProperties: Provided XmlNode's class should be 'Properties'!");
throw error("Provided XmlNode's class should be 'Properties'!");
foreach (XmlNode propNode in propsNode.ChildNodes)
{
@ -17,7 +53,7 @@ namespace RobloxFiles.XmlFormat
XmlNode propName = propNode.Attributes.GetNamedItem("name");
if (propName == null)
throw new Exception("XmlDataReader.ReadProperties: Got a property node without a 'name' attribute!");
throw error("Got a property node without a 'name' attribute!");
IXmlPropertyToken tokenHandler = XmlPropertyTokens.GetHandler(propType);
@ -28,26 +64,28 @@ namespace RobloxFiles.XmlFormat
prop.Instance = instance;
if (!tokenHandler.ReadToken(prop, propNode))
Console.WriteLine("XmlDataReader.ReadProperties: Could not read property: " + prop.GetFullName() + '!');
Console.WriteLine("Could not read property: " + prop.GetFullName() + '!');
instance.AddProperty(ref prop);
}
else
{
Console.WriteLine("XmlDataReader.ReadProperties: No IXmlPropertyToken found for property type: " + propType + '!');
Console.WriteLine("No IXmlPropertyToken found for property type: " + propType + '!');
}
}
}
public static Instance ReadInstance(XmlNode instNode, XmlRobloxFile file = null)
{
var error = createErrorHandler("ReadInstance");
// Process the instance itself
if (instNode.Name != "Item")
throw new Exception("XmlDataReader.ReadInstance: Provided XmlNode's name should be 'Item'!");
throw error("Provided XmlNode's name should be 'Item'!");
XmlNode classToken = instNode.Attributes.GetNamedItem("class");
if (classToken == null)
throw new Exception("XmlDataReader.ReadInstance: Got an Item without a defined 'class' attribute!");
throw error("Got an Item without a defined 'class' attribute!");
Instance inst = new Instance(classToken.InnerText);
@ -59,7 +97,7 @@ namespace RobloxFiles.XmlFormat
string refId = refToken.InnerText;
if (file.Instances.ContainsKey(refId))
throw new Exception("XmlDataReader.ReadInstance: Got an Item with a duplicate 'referent' attribute!");
throw error("Got an Item with a duplicate 'referent' attribute!");
file.Instances.Add(refId, inst);
}

14
XmlFormat/XmlPropertyTokens.cs
View File

@ -37,19 +37,21 @@ namespace RobloxFiles.XmlFormat
public static bool ReadTokenGeneric<T>(Property prop, PropertyType propType, XmlNode token) where T : struct
{
Type resultType = typeof(T);
TypeConverter converter = TypeDescriptor.GetConverter(resultType);
if (converter != null)
try
{
Type resultType = typeof(T);
TypeConverter converter = TypeDescriptor.GetConverter(resultType);
object result = converter.ConvertFromString(token.InnerText);
prop.Type = propType;
prop.Value = result;
return true;
}
return false;
catch
{
return false;
}
}
public static IXmlPropertyToken GetHandler(string tokenName)

44
XmlFormat/XmlRobloxFile.cs
View File

@ -16,6 +16,7 @@ namespace RobloxFiles.XmlFormat
// Runtime Specific
public readonly XmlDocument Root = new XmlDocument();
public Dictionary<string, Instance> Instances = new Dictionary<string, Instance>();
public Dictionary<string, string> SharedStrings = new Dictionary<string, string>();
public void ReadFile(byte[] buffer)
{
@ -50,16 +51,24 @@ namespace RobloxFiles.XmlFormat
Instance item = XmlDataReader.ReadInstance(child, this);
item.Parent = XmlContents;
}
else if (child.Name == "SharedStrings")
{
XmlDataReader.ReadSharedStrings(child, this);
}
}
// Resolve referent properties.
var refProps = Instances.Values
// Query the properties.
var props = Instances.Values
.SelectMany(inst => inst.Properties)
.Where(prop => prop.Type == PropertyType.Ref);
.Select(pair => pair.Value);
// Resolve referent properties.
var refProps = props.Where(prop => prop.Type == PropertyType.Ref);
foreach (Property refProp in refProps)
{
string refId = refProp.Value as string;
if (Instances.ContainsKey(refId))
{
Instance refInst = Instances[refId];
@ -67,13 +76,36 @@ namespace RobloxFiles.XmlFormat
}
else if (refId != "null")
{
Console.WriteLine("XmlRobloxFile: Could not resolve reference for " + refProp.GetFullName());
string name = refProp.GetFullName();
Console.WriteLine("XmlRobloxFile: Could not resolve reference for {0}", name);
}
}
// Resolve shared strings.
var sharedProps = props.Where(prop => prop.Type == PropertyType.SharedString);
foreach (Property sharedProp in sharedProps)
{
string md5 = sharedProp.Value as string;
if (SharedStrings.ContainsKey(md5))
{
string value = SharedStrings[md5];
sharedProp.Value = value;
byte[] data = Convert.FromBase64String(value);
sharedProp.SetRawBuffer(data);
continue;
}
string name = sharedProp.GetFullName();
Console.WriteLine("XmlRobloxFile: Could not resolve shared string for {0}", name);
}
}
else
{
throw new Exception("XmlRobloxFile: No `roblox` tag found!");
throw new Exception("XmlRobloxFile: No 'roblox' tag found!");
}
}
}