Source2Roblox/Roblox-File-Format
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Brought spec up to date, improvements to stability

Browse Source
This commit is contained in:
CloneTrooper1019 2020-07-12 20:19:30 -05:00
parent 7359b6efb7
commit 57fd3f8a25
29 changed files with 551 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
BinaryFormat/BinaryRobloxFile.cs
View File

@ -15,20 +15,22 @@ namespace RobloxFiles
// Header Specific // Header Specific
public const string MagicHeader = "<roblox!\x89\xff\x0d\x0a\x1a\x0a"; public const string MagicHeader = "<roblox!\x89\xff\x0d\x0a\x1a\x0a";
public ushort Version; public ushort Version { get; internal set; }
public uint NumClasses; public uint NumClasses { get; internal set; }
public uint NumInstances; public uint NumInstances { get; internal set; }
public long Reserved; public long Reserved { get; internal set; }
// Runtime Specific // Runtime Specific
public List<BinaryRobloxFileChunk> Chunks = new List<BinaryRobloxFileChunk>(); internal List<BinaryRobloxFileChunk> ChunksImpl = new List<BinaryRobloxFileChunk>();
public IReadOnlyList<BinaryRobloxFileChunk> Chunks => ChunksImpl;
public override string ToString() => GetType().Name; public override string ToString() => GetType().Name;
public Instance[] Instances; public Instance[] Instances { get; internal set; }
public INST[] Classes; public INST[] Classes { get; internal set; }
internal META META = null; internal META META = null;
internal SSTR SSTR = null; internal SSTR SSTR = null;
internal SIGN SIGN = null;
public bool HasMetadata => (META != null); public bool HasMetadata => (META != null);
public Dictionary<string, string> Metadata => META?.Data; public Dictionary<string, string> Metadata => META?.Data;
@ -36,6 +38,9 @@ namespace RobloxFiles
public bool HasSharedStrings => (SSTR != null); public bool HasSharedStrings => (SSTR != null);
public IReadOnlyDictionary<uint, SharedString> SharedStrings => SSTR?.Strings; public IReadOnlyDictionary<uint, SharedString> SharedStrings => SSTR?.Strings;
public bool HasSignatures => (SIGN != null);
public IReadOnlyList<Signature> Signatures => SIGN?.Signatures;
public BinaryRobloxFile() public BinaryRobloxFile()
{ {
Name = "BinaryRobloxFile"; Name = "BinaryRobloxFile";
@ -71,12 +76,10 @@ namespace RobloxFiles
{ {
try try
{ {
BinaryRobloxFileChunk chunk = new BinaryRobloxFileChunk(reader); var chunk = new BinaryRobloxFileChunk(reader);
string chunkType = chunk.ChunkType;
IBinaryFileChunk handler = null; IBinaryFileChunk handler = null;
switch (chunkType) switch (chunk.ChunkType)
{ {
case "INST": case "INST":
handler = new INST(); handler = new INST();
@ -93,22 +96,27 @@ namespace RobloxFiles
case "SSTR": case "SSTR":
handler = new SSTR(); handler = new SSTR();
break; break;
case "SIGN":
handler = new SIGN();
break;
case "END\0": case "END\0":
Chunks.Add(chunk); ChunksImpl.Add(chunk);
reading = false; reading = false;
break; break;
default: case string unhandled:
Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", chunkType); Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", unhandled);
break; break;
default: break;
} }
if (handler != null) if (handler != null)
{ {
using (BinaryRobloxFileReader dataReader = chunk.GetDataReader(this))
handler.LoadFromReader(dataReader);
chunk.Handler = handler; chunk.Handler = handler;
Chunks.Add(chunk);
using (var dataReader = chunk.GetDataReader(this))
handler.Load(dataReader);
ChunksImpl.Add(chunk);
} }
} }
catch (EndOfStreamException) catch (EndOfStreamException)
@ -129,7 +137,7 @@ namespace RobloxFiles
{ {
// Clear the existing data. // Clear the existing data.
Referent = "-1"; Referent = "-1";
Chunks.Clear(); ChunksImpl.Clear();
NumInstances = 0; NumInstances = 0;
NumClasses = 0; NumClasses = 0;
@ -148,7 +156,7 @@ namespace RobloxFiles
// Write the PROP chunks. // Write the PROP chunks.
foreach (INST inst in Classes) foreach (INST inst in Classes)
{ {
Dictionary<string, PROP> props = PROP.CollectProperties(writer, inst); var props = PROP.CollectProperties(writer, inst);
foreach (string propName in props.Keys) foreach (string propName in props.Keys)
{ {
@ -158,26 +166,23 @@ namespace RobloxFiles
} }
// Write the PRNT chunk. // Write the PRNT chunk.
PRNT parents = new PRNT(); var parents = new PRNT();
writer.SaveChunk(parents); writer.SaveChunk(parents);
// Write the SSTR chunk. // Write the SSTR chunk.
if (HasSharedStrings) if (HasSharedStrings)
{ writer.SaveChunk(SSTR, 0);
var sharedStrings = SSTR.SaveAsChunk(writer);
Chunks.Insert(0, sharedStrings);
}
// Write the META chunk. // Write the META chunk.
if (HasMetadata) if (HasMetadata)
{ writer.SaveChunk(META, 0);
var metaChunk = META.SaveAsChunk(writer);
Chunks.Insert(0, metaChunk);
}
// Write the END_ chunk. // Write the SIGN chunk.
var endChunk = writer.WriteEndChunk(); if (HasSignatures)
Chunks.Add(endChunk); writer.SaveChunk(SIGN);
// Write the END chunk.
writer.WriteChunk("END", "</roblox>");
} }
////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////

10
BinaryFormat/Chunks/INST.cs
View File

@ -16,7 +16,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
public override string ToString() => ClassName; public override string ToString() => ClassName;
public void LoadFromReader(BinaryRobloxFileReader reader) public void Load(BinaryRobloxFileReader reader)
{ {
BinaryRobloxFile file = reader.File; BinaryRobloxFile file = reader.File;
@ -59,10 +59,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
file.Classes[ClassIndex] = this; file.Classes[ClassIndex] = this;
} }
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) public void Save(BinaryRobloxFileWriter writer)
{ {
writer.StartWritingChunk(this);
writer.Write(ClassIndex); writer.Write(ClassIndex);
writer.WriteString(ClassName); writer.WriteString(ClassName);
@ -72,7 +70,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
if (IsService) if (IsService)
{ {
BinaryRobloxFile file = writer.File; var file = writer.File;
foreach (int instId in InstanceIds) foreach (int instId in InstanceIds)
{ {
@ -80,8 +78,6 @@ namespace RobloxFiles.BinaryFormat.Chunks
writer.Write(service.Parent == file); writer.Write(service.Parent == file);
} }
} }
return writer.FinishWritingChunk();
} }
} }
} }

15
BinaryFormat/Chunks/META.cs
View File

@ -6,7 +6,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
{ {
public Dictionary<string, string> Data = new Dictionary<string, string>(); public Dictionary<string, string> Data = new Dictionary<string, string>();
public void LoadFromReader(BinaryRobloxFileReader reader) public void Load(BinaryRobloxFileReader reader)
{ {
BinaryRobloxFile file = reader.File; BinaryRobloxFile file = reader.File;
int numEntries = reader.ReadInt32(); int numEntries = reader.ReadInt32();
@ -21,18 +21,15 @@ namespace RobloxFiles.BinaryFormat.Chunks
file.META = this; file.META = this;
} }
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) public void Save(BinaryRobloxFileWriter writer)
{ {
writer.StartWritingChunk(this);
writer.Write(Data.Count); writer.Write(Data.Count);
foreach (var kvPair in Data) foreach (var pair in Data)
{ {
writer.WriteString(kvPair.Key); writer.WriteString(pair.Key);
writer.WriteString(kvPair.Value); writer.WriteString(pair.Value);
} }
return writer.FinishWritingChunk();
} }
} }
} }

56
BinaryFormat/Chunks/PRNT.cs
View File

@ -1,29 +1,29 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
namespace RobloxFiles.BinaryFormat.Chunks namespace RobloxFiles.BinaryFormat.Chunks
{ {
public class PRNT : IBinaryFileChunk public class PRNT : IBinaryFileChunk
{ {
public byte Format { get; private set; } private const byte FORMAT = 0;
public int NumRelations { get; private set; }
public List<int> ChildrenIds { get; private set; } public void Load(BinaryRobloxFileReader reader)
public List<int> ParentIds { get; private set; }
public void LoadFromReader(BinaryRobloxFileReader reader)
{ {
BinaryRobloxFile file = reader.File; BinaryRobloxFile file = reader.File;
Format = reader.ReadByte(); byte format = reader.ReadByte();
NumRelations = reader.ReadInt32(); int idCount = reader.ReadInt32();
ChildrenIds = reader.ReadInstanceIds(NumRelations); if (format != FORMAT)
ParentIds = reader.ReadInstanceIds(NumRelations); throw new Exception($"Unexpected PRNT format: {format} (expected {FORMAT}!)");
for (int i = 0; i < NumRelations; i++) var childIds = reader.ReadInstanceIds(idCount);
var parentIds = reader.ReadInstanceIds(idCount);
for (int i = 0; i < idCount; i++)
{ {
int childId = ChildrenIds[i]; int childId = childIds[i];
int parentId = ParentIds[i]; int parentId = parentIds[i];
Instance child = file.Instances[childId]; Instance child = file.Instances[childId];
child.Parent = (parentId >= 0 ? file.Instances[parentId] : file); child.Parent = (parentId >= 0 ? file.Instances[parentId] : file);
@ -31,15 +31,13 @@ namespace RobloxFiles.BinaryFormat.Chunks
} }
} }
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) public void Save(BinaryRobloxFileWriter writer)
{ {
writer.StartWritingChunk(this); var postInstances = writer.PostInstances;
var idCount = postInstances.Count;
Format = 0; var childIds = new List<int>();
NumRelations = 0; var parentIds = new List<int>();
ChildrenIds = new List<int>();
ParentIds = new List<int>();
foreach (Instance inst in writer.PostInstances) foreach (Instance inst in writer.PostInstances)
{ {
@ -51,19 +49,15 @@ namespace RobloxFiles.BinaryFormat.Chunks
if (parent != null) if (parent != null)
parentId = int.Parse(parent.Referent); parentId = int.Parse(parent.Referent);
ChildrenIds.Add(childId); childIds.Add(childId);
ParentIds.Add(parentId); parentIds.Add(parentId);
NumRelations++;
} }
writer.Write(Format); writer.Write(FORMAT);
writer.Write(NumRelations); writer.Write(idCount);
writer.WriteInstanceIds(ChildrenIds); writer.WriteInstanceIds(childIds);
writer.WriteInstanceIds(ParentIds); writer.WriteInstanceIds(parentIds);
return writer.FinishWritingChunk();
} }
} }
} }

41
BinaryFormat/Chunks/PROP.cs
View File

@ -29,7 +29,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
return $"{Type} {ClassName}.{Name}"; return $"{Type} {ClassName}.{Name}";
} }
public void LoadFromReader(BinaryRobloxFileReader reader) public void Load(BinaryRobloxFileReader reader)
{ {
BinaryRobloxFile file = reader.File; BinaryRobloxFile file = reader.File;
@ -71,7 +71,6 @@ namespace RobloxFiles.BinaryFormat.Chunks
} }
}); });
// Read the property data based on the property type.
switch (Type) switch (Type)
{ {
case PropertyType.String: case PropertyType.String:
@ -500,13 +499,22 @@ namespace RobloxFiles.BinaryFormat.Chunks
readProperties(i => readProperties(i =>
{ {
uint key = SharedKeys[i]; uint key = SharedKeys[i];
return file.SharedStrings[key]; return file.SharedStrings[key];
}); });
break;
case PropertyType.ProtectedString:
readProperties(i =>
{
int length = reader.ReadInt32();
byte[] buffer = reader.ReadBytes(length);
return new ProtectedString(buffer);
});
break; break;
default: default:
Console.WriteLine("Unhandled property type: {0}!", Type); Console.Error.WriteLine("Unhandled property type: {0}!", Type);
break; break;
// //
} }
@ -546,7 +554,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
return propMap; return propMap;
} }
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) public void Save(BinaryRobloxFileWriter writer)
{ {
BinaryRobloxFile file = writer.File; BinaryRobloxFile file = writer.File;
@ -567,9 +575,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.Add(prop); props.Add(prop);
} }
writer.StartWritingChunk(this);
writer.Write(ClassIndex); writer.Write(ClassIndex);
writer.WriteString(Name); writer.WriteString(Name);
writer.Write(TypeId); writer.Write(TypeId);
@ -996,12 +1002,12 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.ForEach(prop => props.ForEach(prop =>
{ {
SharedString shared = prop.CastValue<SharedString>(); var shared = prop.CastValue<SharedString>();
string key = shared.MD5_Key; string key = shared.Key;
if (!sstr.Lookup.ContainsKey(key)) if (!sstr.Lookup.ContainsKey(key))
{ {
uint id = (uint)(sstr.NumHashes++); uint id = (uint)(sstr.Lookup.Count);
sstr.Strings.Add(id, shared); sstr.Strings.Add(id, shared);
sstr.Lookup.Add(key, id); sstr.Lookup.Add(key, id);
} }
@ -1012,10 +1018,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
writer.WriteInterleaved(sharedKeys); writer.WriteInterleaved(sharedKeys);
break; break;
// case PropertyType.ProtectedString:
} props.ForEach(prop =>
{
var protect = prop.CastValue<ProtectedString>();
byte[] buffer = protect.RawBuffer;
return writer.FinishWritingChunk(); writer.Write(buffer.Length);
writer.Write(buffer);
});
break;
default: break;
}
} }
} }
} }

56
BinaryFormat/Chunks/SIGN.cs Normal file
View File

@ -0,0 +1,56 @@
namespace RobloxFiles.BinaryFormat.Chunks
{
public struct Signature
{
public int Version;
public long Id;
public int Length;
public byte[] Data;
}
public class SIGN : IBinaryFileChunk
{
public Signature[] Signatures;
public void Load(BinaryRobloxFileReader reader)
{
int numSignatures = reader.ReadInt32();
Signatures = new Signature[numSignatures];
for (int i = 0; i < numSignatures; i++)
{
var signature = new Signature
{
Version = reader.ReadInt32(),
Id = reader.ReadInt64(),
Length = reader.ReadInt32(),
};
signature.Data = reader.ReadBytes(signature.Length);
Signatures[i] = signature;
}
var file = reader.File;
file.SIGN = this;
}
public void Save(BinaryRobloxFileWriter writer)
{
writer.Write(Signatures.Length);
for (int i = 0; i < Signatures.Length; i++)
{
var signature = Signatures[i];
signature.Length = signature.Data.Length;
writer.Write(signature.Version);
writer.Write(signature.Id);
writer.Write(signature.Length);
writer.Write(signature.Data);
}
}
}
}

44
BinaryFormat/Chunks/SSTR.cs
View File

@ -6,56 +6,54 @@ namespace RobloxFiles.BinaryFormat.Chunks
{ {
public class SSTR : IBinaryFileChunk public class SSTR : IBinaryFileChunk
{ {
public int Version; private const int FORMAT = 0;
public int NumHashes;
public Dictionary<string, uint> Lookup = new Dictionary<string, uint>(); internal Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
public Dictionary<uint, SharedString> Strings = new Dictionary<uint, SharedString>(); internal Dictionary<uint, SharedString> Strings = new Dictionary<uint, SharedString>();
public void LoadFromReader(BinaryRobloxFileReader reader) public void Load(BinaryRobloxFileReader reader)
{ {
BinaryRobloxFile file = reader.File; BinaryRobloxFile file = reader.File;
Version = reader.ReadInt32(); int format = reader.ReadInt32();
NumHashes = reader.ReadInt32(); int numHashes = reader.ReadInt32();
for (uint id = 0; id < NumHashes; id++) if (format != FORMAT)
throw new Exception($"Unexpected SSTR format: {format} (expected {FORMAT}!)");
for (uint id = 0; id < numHashes; id++)
{ {
byte[] md5 = reader.ReadBytes(16); byte[] hash = reader.ReadBytes(16);
string key = Convert.ToBase64String(hash);
int length = reader.ReadInt32();
byte[] data = reader.ReadBytes(length);
byte[] data = reader.ReadBuffer();
SharedString value = SharedString.FromBuffer(data); SharedString value = SharedString.FromBuffer(data);
Lookup.Add(value.MD5_Key, id);
Lookup.Add(key, id);
Strings.Add(id, value); Strings.Add(id, value);
} }
file.SSTR = this; file.SSTR = this;
} }
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) public void Save(BinaryRobloxFileWriter writer)
{ {
writer.StartWritingChunk(this); writer.Write(FORMAT);
writer.Write(Lookup.Count);
writer.Write(Version);
writer.Write(NumHashes);
foreach (var pair in Lookup) foreach (var pair in Lookup)
{ {
string key = pair.Key; string key = pair.Key;
byte[] md5 = Convert.FromBase64String(key); byte[] hash = Convert.FromBase64String(key);
writer.Write(md5); writer.Write(hash);
SharedString value = Strings[pair.Value]; SharedString value = Strings[pair.Value];
byte[] buffer = SharedString.FindRecord(value.MD5_Key); byte[] buffer = SharedString.Find(value.Key);
writer.Write(buffer.Length); writer.Write(buffer.Length);
writer.Write(buffer); writer.Write(buffer);
} }
return writer.FinishWritingChunk();
} }
} }
} }

59
BinaryFormat/IO/BinaryFileWriter.cs
View File

@ -1,6 +1,5 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
@ -18,14 +17,16 @@ namespace RobloxFiles.BinaryFormat
public string ChunkType { get; private set; } public string ChunkType { get; private set; }
public long ChunkStart { get; private set; } public long ChunkStart { get; private set; }
public Dictionary<string, INST> ClassMap;
public readonly BinaryRobloxFile File; public readonly BinaryRobloxFile File;
// Dictionary mapping ClassNames to their INST chunks.
private readonly Dictionary<string, INST> ClassMap;
// Instances in parent->child order // Instances in parent->child order
public List<Instance> Instances; private readonly List<Instance> Instances;
// Instances in child->parent order // Instances in child->parent order
public List<Instance> PostInstances; internal List<Instance> PostInstances { get; private set; }
public BinaryRobloxFileWriter(BinaryRobloxFile file, Stream workBuffer = null) : base(workBuffer ?? new MemoryStream()) public BinaryRobloxFileWriter(BinaryRobloxFile file, Stream workBuffer = null) : base(workBuffer ?? new MemoryStream())
{ {
@ -104,13 +105,13 @@ namespace RobloxFiles.BinaryFormat
} }
// Encodes an int for an interleaved buffer. // Encodes an int for an interleaved buffer.
private int EncodeInt(int value) private static int EncodeInt(int value)
{ {
return (value << 1) ^ (value >> 31); return (value << 1) ^ (value >> 31);
} }
// Encodes a float for an interleaved buffer. // Encodes a float for an interleaved buffer.
private float EncodeFloat(float value) private static float EncodeFloat(float value)
{ {
byte[] buffer = BitConverter.GetBytes(value); byte[] buffer = BitConverter.GetBytes(value);
uint bits = BitConverter.ToUInt32(buffer, 0); uint bits = BitConverter.ToUInt32(buffer, 0);
@ -171,9 +172,9 @@ namespace RobloxFiles.BinaryFormat
Instances.Add(instance); Instances.Add(instance);
string className = instance.ClassName; string className = instance.ClassName;
INST inst = null; INST inst;
if (!ClassMap.ContainsKey(className)) if (!ClassMap.TryGetValue(className, out inst))
{ {
inst = new INST() inst = new INST()
{ {
@ -184,10 +185,6 @@ namespace RobloxFiles.BinaryFormat
ClassMap.Add(className, inst); ClassMap.Add(className, inst);
} }
else
{
inst = ClassMap[className];
}
inst.NumInstances++; inst.NumInstances++;
inst.InstanceIds.Add(instId); inst.InstanceIds.Add(instId);
@ -222,7 +219,7 @@ namespace RobloxFiles.BinaryFormat
} }
// Marks that we are writing a chunk. // Marks that we are writing a chunk.
public bool StartWritingChunk(string chunkType) private bool StartWritingChunk(string chunkType)
{ {
if (chunkType.Length != 4) if (chunkType.Length != 4)
throw new Exception("BinaryFileWriter.StartWritingChunk - ChunkType length should be 4!"); throw new Exception("BinaryFileWriter.StartWritingChunk - ChunkType length should be 4!");
@ -241,7 +238,7 @@ namespace RobloxFiles.BinaryFormat
} }
// Marks that we are writing a chunk. // Marks that we are writing a chunk.
public bool StartWritingChunk(IBinaryFileChunk chunk) private bool StartWritingChunk(IBinaryFileChunk chunk)
{ {
if (!WritingChunk) if (!WritingChunk)
{ {
@ -257,7 +254,7 @@ namespace RobloxFiles.BinaryFormat
} }
// Compresses the data that was written into a BinaryRobloxFileChunk and writes it. // Compresses the data that was written into a BinaryRobloxFileChunk and writes it.
public BinaryRobloxFileChunk FinishWritingChunk(bool compress = true) private BinaryRobloxFileChunk FinishWritingChunk(bool compress = true)
{ {
if (!WritingChunk) if (!WritingChunk)
throw new Exception($"BinaryRobloxFileWriter: Cannot finish writing a chunk without starting one!"); throw new Exception($"BinaryRobloxFileWriter: Cannot finish writing a chunk without starting one!");
@ -283,18 +280,36 @@ namespace RobloxFiles.BinaryFormat
return chunk; return chunk;
} }
public void SaveChunk(IBinaryFileChunk handler) internal BinaryRobloxFileChunk SaveChunk(IBinaryFileChunk handler, int insertPos = -1)
{ {
var chunk = handler.SaveAsChunk(this); StartWritingChunk(handler);
File.Chunks.Add(chunk); handler.Save(this);
var chunk = FinishWritingChunk();
if (insertPos >= 0)
File.ChunksImpl.Insert(insertPos, chunk);
else
File.ChunksImpl.Add(chunk);
return chunk;
} }
public BinaryRobloxFileChunk WriteEndChunk() internal BinaryRobloxFileChunk WriteChunk(string chunkType, string content, bool compress = false)
{ {
StartWritingChunk("END\0"); if (chunkType.Length > 4)
WriteString("</roblox>", true); chunkType = chunkType.Substring(0, 4);
return FinishWritingChunk(false); while (chunkType.Length < 4)
chunkType += '\0';
StartWritingChunk(chunkType);
WriteString(content);
var chunk = FinishWritingChunk(compress);
File.ChunksImpl.Add(chunk);
return chunk;
} }
} }
} }

41
DataTypes/ProtectedString.cs
View File

@ -1,27 +1,54 @@
namespace RobloxFiles.DataTypes using System.Text;
namespace RobloxFiles.DataTypes
{ {
/// <summary> /// <summary>
/// ProtectedString is a type used by some of the XML properties. /// ProtectedString is a type used by the Source property of scripts.
/// Here, it only exists as a wrapper class for strings. /// If constructed as an array of bytes, it's assumed to be compiled byte-code.
/// </summary> /// </summary>
public class ProtectedString public class ProtectedString
{ {
public readonly string ProtectedValue; public readonly bool IsCompiled;
public override string ToString() => ProtectedValue; public readonly byte[] RawBuffer;
public override string ToString()
{
if (IsCompiled)
return $"byte[{RawBuffer.Length}]";
return Encoding.UTF8.GetString(RawBuffer);
}
public ProtectedString(string value) public ProtectedString(string value)
{ {
ProtectedValue = value; IsCompiled = false;
RawBuffer = Encoding.UTF8.GetBytes(value);
}
public ProtectedString(byte[] compiled)
{
IsCompiled = true;
RawBuffer = compiled;
} }
public static implicit operator string(ProtectedString protectedString) public static implicit operator string(ProtectedString protectedString)
{ {
return protectedString.ProtectedValue; return Encoding.UTF8.GetString(protectedString.RawBuffer);
} }
public static implicit operator ProtectedString(string value) public static implicit operator ProtectedString(string value)
{ {
return new ProtectedString(value); return new ProtectedString(value);
} }
public static implicit operator byte[](ProtectedString protectedString)
{
return protectedString.RawBuffer;
}
public static implicit operator ProtectedString(byte[] value)
{
return new ProtectedString(value);
}
} }
} }

47
DataTypes/SharedString.cs
View File

@ -1,43 +1,58 @@
using System; using System;
using System.Collections.Generic;
using System.Text; using System.Text;
using System.Security.Cryptography; using System.Collections.Generic;
using Konscious.Security.Cryptography;
namespace RobloxFiles.DataTypes namespace RobloxFiles.DataTypes
{ {
// SharedString is a datatype that takes a sequence of bytes and stores it in a
// lookup table that is shared by the entire file. It originally used MD5 for the
// hashing, but Roblox now uses Blake2B to avoid the obvious problems with using MD5.
// In practice the value of a SharedString does not have to match the hash of the
// data it represents, it just needs to be distinct and MUST be 16 bytes long.
// The XML format still uses 'md5' as its attribute key to the lookup table.
public class SharedString public class SharedString
{ {
private static Dictionary<string, byte[]> Records = new Dictionary<string, byte[]>(); private static Dictionary<string, byte[]> Lookup = new Dictionary<string, byte[]>();
public readonly string MD5_Key; public string Key { get; internal set; }
public string ComputedKey { get; internal set; }
public byte[] SharedValue => FindRecord(MD5_Key); public byte[] SharedValue => Find(ComputedKey ?? Key);
public override string ToString() => $"MD5 Key: {MD5_Key}"; public override string ToString() => $"Key: {ComputedKey ?? Key}";
internal SharedString(string md5) internal SharedString(string key)
{ {
MD5_Key = md5; Key = key;
}
internal static void Register(string key, byte[] buffer)
{
Lookup.Add(key, buffer);
} }
private SharedString(byte[] buffer) private SharedString(byte[] buffer)
{ {
using (MD5 md5 = MD5.Create()) using (HMACBlake2B blake2B = new HMACBlake2B(16 * 8))
{ {
byte[] hash = md5.ComputeHash(buffer); byte[] hash = blake2B.ComputeHash(buffer);
MD5_Key = Convert.ToBase64String(hash); ComputedKey = Convert.ToBase64String(hash);
Key = ComputedKey;
} }
if (Records.ContainsKey(MD5_Key)) if (Lookup.ContainsKey(ComputedKey))
return; return;
Records.Add(MD5_Key, buffer); Register(ComputedKey, buffer);
} }
public static byte[] FindRecord(string key) public static byte[] Find(string key)
{ {
byte[] result = null; byte[] result = null;
if (Records.ContainsKey(key)) if (Lookup.ContainsKey(key))
result = Records[key]; result = Lookup[key];
return result; return result;
} }

4
Interfaces/IBinaryFileChunk.cs
View File

@ -2,7 +2,7 @@
{ {
public interface IBinaryFileChunk public interface IBinaryFileChunk
{ {
void LoadFromReader(BinaryRobloxFileReader reader); void Load(BinaryRobloxFileReader reader);
BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer); void Save(BinaryRobloxFileWriter writer);
} }
} }

47
RobloxFile.cs
View File

@ -6,17 +6,23 @@ using System.Threading.Tasks;
namespace RobloxFiles namespace RobloxFiles
{ {
/// <summary> /// <summary>
/// Represents a loaded *.rbxl/*.rbxm Roblox file. /// Represents a loaded Roblox place/model file.
/// The contents of the RobloxFile are stored as its children. /// RobloxFile is an Instance and its children are the contents of the file.
/// </summary> /// </summary>
public abstract class RobloxFile : Instance public abstract class RobloxFile : Instance
{ {
protected abstract void ReadFile(byte[] buffer); protected abstract void ReadFile(byte[] buffer);
/// <summary>
/// Saves this RobloxFile to the provided stream.
/// </summary>
/// <param name="stream">The stream to save to.</param>
public abstract void Save(Stream stream); public abstract void Save(Stream stream);
/// <summary> /// <summary>
/// Opens a RobloxFile using the provided buffer. /// Opens a RobloxFile using the provided buffer.
/// </summary> /// </summary>
/// <returns>The opened RobloxFile.</returns>
public static RobloxFile Open(byte[] buffer) public static RobloxFile Open(byte[] buffer)
{ {
if (buffer.Length > 14) if (buffer.Length > 14)
@ -43,6 +49,7 @@ namespace RobloxFiles
/// Opens a Roblox file by reading from a provided Stream. /// Opens a Roblox file by reading from a provided Stream.
/// </summary> /// </summary>
/// <param name="stream">The stream to read the Roblox file from.</param> /// <param name="stream">The stream to read the Roblox file from.</param>
/// <returns>The opened RobloxFile.</returns>
public static RobloxFile Open(Stream stream) public static RobloxFile Open(Stream stream)
{ {
byte[] buffer; byte[] buffer;
@ -60,6 +67,7 @@ namespace RobloxFiles
/// Opens a Roblox file from a provided file path. /// Opens a Roblox file from a provided file path.
/// </summary> /// </summary>
/// <param name="filePath">A path to a Roblox file to be opened.</param> /// <param name="filePath">A path to a Roblox file to be opened.</param>
/// <returns>The opened RobloxFile.</returns>
public static RobloxFile Open(string filePath) public static RobloxFile Open(string filePath)
{ {
byte[] buffer = File.ReadAllBytes(filePath); byte[] buffer = File.ReadAllBytes(filePath);
@ -70,6 +78,7 @@ namespace RobloxFiles
/// Creates and runs a Task to open a Roblox file from a byte sequence that represents the file. /// Creates and runs a Task to open a Roblox file from a byte sequence that represents the file.
/// </summary> /// </summary>
/// <param name="buffer">A byte sequence that represents the file.</param> /// <param name="buffer">A byte sequence that represents the file.</param>
/// <returns>A task which will complete once the file is opened with the resulting RobloxFile.</returns>
public static Task<RobloxFile> OpenAsync(byte[] buffer) public static Task<RobloxFile> OpenAsync(byte[] buffer)
{ {
return Task.Run(() => Open(buffer)); return Task.Run(() => Open(buffer));
@ -79,6 +88,7 @@ namespace RobloxFiles
/// Creates and runs a Task to open a Roblox file using a provided Stream. /// Creates and runs a Task to open a Roblox file using a provided Stream.
/// </summary> /// </summary>
/// <param name="stream">The stream to read the Roblox file from.</param> /// <param name="stream">The stream to read the Roblox file from.</param>
/// <returns>A task which will complete once the file is opened with the resulting RobloxFile.</returns>
public static Task<RobloxFile> OpenAsync(Stream stream) public static Task<RobloxFile> OpenAsync(Stream stream)
{ {
return Task.Run(() => Open(stream)); return Task.Run(() => Open(stream));
@ -88,9 +98,42 @@ namespace RobloxFiles
/// Opens a Roblox file from a provided file path. /// Opens a Roblox file from a provided file path.
/// </summary> /// </summary>
/// <param name="filePath">A path to a Roblox file to be opened.</param> /// <param name="filePath">A path to a Roblox file to be opened.</param>
/// <returns>A task which will complete once the file is opened with the resulting RobloxFile.</returns>
public static Task<RobloxFile> OpenAsync(string filePath) public static Task<RobloxFile> OpenAsync(string filePath)
{ {
return Task.Run(() => Open(filePath)); return Task.Run(() => Open(filePath));
} }
/// <summary>
/// Saves this RobloxFile to the provided file path.
/// </summary>
/// <param name="filePath">A path to where the file should be saved.</param>
public void Save(string filePath)
{
using (FileStream stream = File.OpenWrite(filePath))
{
Save(stream);
}
}
/// <summary>
/// Asynchronously saves this RobloxFile to the provided stream.
/// </summary>
/// <param name="stream">The stream to save to.</param>
/// <returns>A task which will complete upon the save's completion.</returns>
public Task SaveAsync(Stream stream)
{
return Task.Run(() => Save(stream));
}
/// <summary>
/// Asynchronously saves this RobloxFile to the provided file path.
/// </summary>
/// <param name="filePath">A path to where the file should be saved.</param>
/// <returns>A task which will complete upon the save's completion.</returns>
public Task SaveAsync(string filePath)
{
return Task.Run(() => Save(filePath));
}
} }
} }

28
RobloxFileFormat.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="packages\Costura.Fody.3.3.3\build\Costura.Fody.props" Condition="Exists('packages\Costura.Fody.3.3.3\build\Costura.Fody.props')" /> <Import Project="packages\Costura.Fody.4.1.0\build\Costura.Fody.props" Condition="Exists('packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -52,9 +52,11 @@
<StartupObject /> <StartupObject />
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="Costura, Version=3.3.3.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL"> <Reference Include="Costura, Version=4.1.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
<HintPath>packages\Costura.Fody.3.3.3\lib\net40\Costura.dll</HintPath> <HintPath>packages\Costura.Fody.4.1.0\lib\net40\Costura.dll</HintPath>
<Private>True</Private> </Reference>
<Reference Include="Konscious.Security.Cryptography.Blake2, Version=1.0.9.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>packages\Konscious.Security.Cryptography.Blake2.1.0.9\lib\net46\Konscious.Security.Cryptography.Blake2.dll</HintPath>
</Reference> </Reference>
<Reference Include="LZ4, Version=1.0.15.93, Culture=neutral, PublicKeyToken=62e1b5ec1eec9bdd, processorArchitecture=MSIL"> <Reference Include="LZ4, Version=1.0.15.93, Culture=neutral, PublicKeyToken=62e1b5ec1eec9bdd, processorArchitecture=MSIL">
<HintPath>packages\lz4net.1.0.15.93\lib\net4-client\LZ4.dll</HintPath> <HintPath>packages\lz4net.1.0.15.93\lib\net4-client\LZ4.dll</HintPath>
@ -62,6 +64,10 @@
</Reference> </Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core" /> <Reference Include="System.Core" />
<Reference Include="System.Numerics" />
<Reference Include="System.Numerics.Vectors, Version=4.1.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
</Reference>
<Reference Include="System.Xml.Linq" /> <Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" /> <Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" /> <Reference Include="Microsoft.CSharp" />
@ -76,6 +82,7 @@
<Compile Include="BinaryFormat\Chunks\META.cs" /> <Compile Include="BinaryFormat\Chunks\META.cs" />
<Compile Include="BinaryFormat\Chunks\PRNT.cs" /> <Compile Include="BinaryFormat\Chunks\PRNT.cs" />
<Compile Include="BinaryFormat\Chunks\PROP.cs" /> <Compile Include="BinaryFormat\Chunks\PROP.cs" />
<Compile Include="BinaryFormat\Chunks\SIGN.cs" />
<Compile Include="BinaryFormat\Chunks\SSTR.cs" /> <Compile Include="BinaryFormat\Chunks\SSTR.cs" />
<Compile Include="BinaryFormat\IO\BinaryFileReader.cs" /> <Compile Include="BinaryFormat\IO\BinaryFileReader.cs" />
<Compile Include="BinaryFormat\IO\BinaryFileWriter.cs" /> <Compile Include="BinaryFormat\IO\BinaryFileWriter.cs" />
@ -88,7 +95,6 @@
<Compile Include="Interfaces\IBinaryFileChunk.cs" /> <Compile Include="Interfaces\IBinaryFileChunk.cs" />
<Compile Include="Generated\Classes.cs" /> <Compile Include="Generated\Classes.cs" />
<Compile Include="Generated\Enums.cs" /> <Compile Include="Generated\Enums.cs" />
<Compile Include="LibTest\Program.cs" />
<Compile Include="Tree\Attributes.cs" /> <Compile Include="Tree\Attributes.cs" />
<Compile Include="Tree\Property.cs" /> <Compile Include="Tree\Property.cs" />
<Compile Include="Tree\Instance.cs" /> <Compile Include="Tree\Instance.cs" />
@ -167,24 +173,18 @@
</BootstrapperPackage> </BootstrapperPackage>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="LibTest\Binary.rbxl">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Include="LibTest\Xml.rbxlx">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Include="packages.config" /> <None Include="packages.config" />
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<PropertyGroup> <PropertyGroup>
<PostBuildEvent>copy /y $(TargetPath) $(ProjectDir)$(TargetFileName)</PostBuildEvent> <PostBuildEvent>copy /y $(TargetPath) $(ProjectDir)$(TargetFileName)</PostBuildEvent>
</PropertyGroup> </PropertyGroup>
<Import Project="packages\Fody.4.2.1\build\Fody.targets" Condition="Exists('packages\Fody.4.2.1\build\Fody.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('packages\Fody.4.2.1\build\Fody.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Fody.4.2.1\build\Fody.targets'))" /> <Error Condition="!Exists('packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Costura.Fody.4.1.0\build\Costura.Fody.props'))" />
<Error Condition="!Exists('packages\Costura.Fody.3.3.3\build\Costura.Fody.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Costura.Fody.3.3.3\build\Costura.Fody.props'))" /> <Error Condition="!Exists('packages\Fody.6.2.0\build\Fody.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Fody.6.2.0\build\Fody.targets'))" />
</Target> </Target>
<Import Project="packages\Fody.6.2.0\build\Fody.targets" Condition="Exists('packages\Fody.6.2.0\build\Fody.targets')" />
</Project> </Project>

BIN
RobloxFileFormat.dll
View File

Binary file not shown.

6
RobloxFileFormat.sln
View File

@ -5,6 +5,8 @@ VisualStudioVersion = 16.0.29920.165
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobloxFileFormat", "RobloxFileFormat.csproj", "{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobloxFileFormat", "RobloxFileFormat.csproj", "{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobloxFileFormat.UnitTest", "UnitTest\RobloxFileFormat.UnitTest.csproj", "{6BCA31B2-58D8-4689-9929-88E16040BF29}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@ -15,6 +17,10 @@ Global
{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Debug|Any CPU.Build.0 = Debug|Any CPU {CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.ActiveCfg = Release|Any CPU {CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.Build.0 = Release|Any CPU {CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.Build.0 = Release|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

2
Tree/Attributes.cs
View File

@ -54,7 +54,7 @@ namespace RobloxFiles
} }
internal BinaryReader reader; internal BinaryReader reader;
internal BinaryWriter writer; // internal BinaryWriter writer;
internal int readInt() => reader.ReadInt32(); internal int readInt() => reader.ReadInt32();
internal byte readByte() => reader.ReadByte(); internal byte readByte() => reader.ReadByte();

6
Tree/Instance.cs
View File

@ -521,10 +521,10 @@ namespace RobloxFiles
PropertyType propType = PropertyType.Unknown; PropertyType propType = PropertyType.Unknown;
if (fieldType.IsEnum) if (Property.Types.ContainsKey(fieldType))
propType = PropertyType.Enum;
else if (Property.Types.ContainsKey(fieldType))
propType = Property.Types[fieldType]; propType = Property.Types[fieldType];
else if (fieldType.IsEnum)
propType = PropertyType.Enum;
if (propType != PropertyType.Unknown) if (propType != PropertyType.Unknown)
{ {

28
Tree/Property.cs
View File

@ -42,6 +42,7 @@ namespace RobloxFiles
Color3uint8, Color3uint8,
Int64, Int64,
SharedString, SharedString,
ProtectedString
} }
public class Property public class Property
@ -60,6 +61,7 @@ namespace RobloxFiles
internal static BindingFlags BindingFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.FlattenHierarchy | BindingFlags.IgnoreCase; internal static BindingFlags BindingFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.FlattenHierarchy | BindingFlags.IgnoreCase;
internal static MemberTypes FieldOrProperty = MemberTypes.Field | MemberTypes.Property; internal static MemberTypes FieldOrProperty = MemberTypes.Field | MemberTypes.Property;
// !! FIXME: Map typeof(ProtectedString) to PropertyType.ProtectedString when binary files are allowed to read it.
public static readonly IReadOnlyDictionary<Type, PropertyType> Types = new Dictionary<Type, PropertyType>() public static readonly IReadOnlyDictionary<Type, PropertyType> Types = new Dictionary<Type, PropertyType>()
{ {
{ typeof(Axes), PropertyType.Axes }, { typeof(Axes), PropertyType.Axes },
@ -98,17 +100,23 @@ namespace RobloxFiles
private void ImproviseRawBuffer() private void ImproviseRawBuffer()
{ {
if (RawValue is byte[]) if (RawValue is SharedString)
{
RawBuffer = RawValue as byte[];
return;
}
else if (RawValue is SharedString)
{ {
var sharedString = CastValue<SharedString>(); var sharedString = CastValue<SharedString>();
RawBuffer = sharedString.SharedValue; RawBuffer = sharedString.SharedValue;
return; return;
} }
else if (RawValue is ProtectedString)
{
var protectedString = CastValue<ProtectedString>();
RawBuffer = protectedString.RawBuffer;
return;
}
else if (RawValue is byte[])
{
RawBuffer = RawValue as byte[];
return;
}
switch (Type) switch (Type)
{ {
@ -127,7 +135,7 @@ namespace RobloxFiles
case PropertyType.Double: case PropertyType.Double:
RawBuffer = BitConverter.GetBytes((double)Value); RawBuffer = BitConverter.GetBytes((double)Value);
break; break;
// default: break;
} }
} }
@ -142,7 +150,11 @@ namespace RobloxFiles
if (typeName == Name) if (typeName == Name)
{ {
FieldInfo directField = instType.GetField(typeName, BindingFlags.DeclaredOnly); FieldInfo directField = instType
.GetFields()
.Where(field => field.Name.StartsWith(Name))
.Where(field => field.DeclaringType == instType)
.FirstOrDefault();
if (directField != null) if (directField != null)
{ {

0
LibTest/Binary.rbxl → UnitTest/Files/Binary.rbxl
View File

0
LibTest/Xml.rbxlx → UnitTest/Files/Xml.rbxlx
View File

71
LibTest/Program.cs → UnitTest/Program.cs
View File

@ -2,19 +2,59 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using RobloxFiles.DataTypes; using RobloxFiles.DataTypes;
namespace RobloxFiles namespace RobloxFiles.UnitTest
{ {
// If the solution is built as an exe, this class is static class Program
// used to drive some basic testing of the library.
internal static class Program
{ {
const string pattern = "\\d+$"; const string pattern = "\\d+$";
static void PrintTreeImpl(Instance inst, int stack = 0)
{
string padding = "";
string extension = "";
for (int i = 0; i < stack; i++)
padding += '\t';
switch (inst.ClassName)
{
case "Script":
extension = ".server.lua";
break;
case "LocalScript":
extension = ".client.lua";
break;
case "ModuleScript":
extension = ".lua";
break;
default: break;
}
Console.WriteLine($"{padding}{inst.Name}{extension}");
var children = inst
.GetChildren()
.ToList();
children.ForEach(child => PrintTreeImpl(child, stack + 1));
}
static void PrintTree(string path)
{
Console.WriteLine("Opening file...");
RobloxFile target = RobloxFile.Open(path);
foreach (Instance child in target.GetChildren())
PrintTreeImpl(child);
Debugger.Break();
}
static void CountAssets(string path) static void CountAssets(string path)
{ {
Console.WriteLine("Opening file..."); Console.WriteLine("Opening file...");
@ -23,6 +63,14 @@ namespace RobloxFiles
var workspace = target.FindFirstChildOfClass<Workspace>(); var workspace = target.FindFirstChildOfClass<Workspace>();
var assets = new HashSet<string>(); var assets = new HashSet<string>();
if (workspace == null)
{
Console.WriteLine("No workspace found!");
Debugger.Break();
return;
}
foreach (Instance inst in workspace.GetDescendants()) foreach (Instance inst in workspace.GetDescendants())
{ {
var instPath = inst.GetFullName(); var instPath = inst.GetFullName();
@ -63,21 +111,18 @@ namespace RobloxFiles
if (args.Length > 0) if (args.Length > 0)
{ {
string path = args[0]; string path = args[0];
CountAssets(path); PrintTree(path);
} }
else else
{ {
RobloxFile bin = RobloxFile.Open(@"LibTest\Binary.rbxl"); RobloxFile bin = RobloxFile.Open(@"Files\Binary.rbxl");
RobloxFile xml = RobloxFile.Open(@"LibTest\Xml.rbxlx"); RobloxFile xml = RobloxFile.Open(@"Files\Xml.rbxlx");
Console.WriteLine("Files opened! Pausing execution for debugger analysis..."); Console.WriteLine("Files opened! Pausing execution for debugger analysis...");
Debugger.Break(); Debugger.Break();
using (FileStream binStream = File.OpenWrite(@"LibTest\Binary_SaveTest.rbxl")) bin.Save(@"Files\Binary_SaveTest.rbxl");
bin.Save(binStream); xml.Save(@"Files\Xml_SaveTest.rbxlx");
using (FileStream xmlStream = File.OpenWrite(@"LibTest\Xml_SaveTest.rbxlx"))
xml.Save(xmlStream);
Console.WriteLine("Files saved! Pausing execution for debugger analysis..."); Console.WriteLine("Files saved! Pausing execution for debugger analysis...");
Debugger.Break(); Debugger.Break();

31
UnitTest/RobloxFileFormat.UnitTest.csproj Normal file
View File

@ -0,0 +1,31 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>WinExe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<None Remove="Files\Binary.rbxl" />
<None Remove="Files\CoreScripts.rbxm" />
<None Remove="Files\Xml.rbxlx" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Files\Binary.rbxl">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="Files\Xml.rbxlx">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\RobloxFileFormat.csproj" />
</ItemGroup>
</Project>

10
Utility/Formatting.cs
View File

@ -115,14 +115,18 @@ internal static class Formatting
return Math.Abs(a - b) < epsilon; return Math.Abs(a - b) < epsilon;
} }
public static byte[] ReadBuffer(this BinaryReader reader)
{
int len = reader.ReadInt32();
return reader.ReadBytes(len);
}
public static string ReadString(this BinaryReader reader, bool useIntLength) public static string ReadString(this BinaryReader reader, bool useIntLength)
{ {
if (!useIntLength) if (!useIntLength)
return reader.ReadString(); return reader.ReadString();
int len = reader.ReadInt32(); byte[] buffer = reader.ReadBuffer();
byte[] buffer = reader.ReadBytes(len);
return Encoding.UTF8.GetString(buffer); return Encoding.UTF8.GetString(buffer);
} }
} }

22
XmlFormat/IO/XmlFileReader.cs
View File

@ -11,7 +11,7 @@ namespace RobloxFiles.XmlFormat
{ {
var errorHandler = new Func<string, Exception>((message) => var errorHandler = new Func<string, Exception>((message) =>
{ {
string contents = $"XmlDataReader.{label}: {message}"; string contents = $"XmlRobloxFileReader.{label}: {message}";
return new Exception(contents); return new Exception(contents);
}); });
@ -29,19 +29,25 @@ namespace RobloxFiles.XmlFormat
{ {
if (sharedString.Name == "SharedString") if (sharedString.Name == "SharedString")
{ {
XmlNode md5Node = sharedString.Attributes.GetNamedItem("md5"); XmlNode hashNode = sharedString.Attributes.GetNamedItem("md5");
if (md5Node == null) if (hashNode == null)
throw error("Got a SharedString without an 'md5' attribute!"); throw error("Got a SharedString without an 'md5' attribute!");
string key = md5Node.InnerText; string key = hashNode.InnerText;
string value = sharedString.InnerText.Replace("\n", ""); string value = sharedString.InnerText.Replace("\n", "");
byte[] buffer = Convert.FromBase64String(value); byte[] hash = Convert.FromBase64String(key);
SharedString record = SharedString.FromBase64(value); var record = SharedString.FromBase64(value);
if (record.MD5_Key != key) if (hash.Length != 16)
throw error("The provided md5 hash did not match with the md5 hash computed for the value!"); throw error($"SharedString base64 key '{key}' must decode to byte[16]!");
if (key != record.Key)
{
SharedString.Register(key, record.SharedValue);
record.Key = key;
}
file.SharedStrings.Add(key); file.SharedStrings.Add(key);
} }

40
XmlFormat/IO/XmlFileWriter.cs
View File

@ -1,4 +1,5 @@
using System; using System;
using System.Linq;
using System.Text; using System.Text;
using System.Xml; using System.Xml;
@ -47,6 +48,9 @@ namespace RobloxFiles.XmlFormat
public static XmlNode WriteProperty(Property prop, XmlDocument doc, XmlRobloxFile file) public static XmlNode WriteProperty(Property prop, XmlDocument doc, XmlRobloxFile file)
{ {
if (prop.Name == "Archivable")
return null;
string propType = prop.XmlToken; string propType = prop.XmlToken;
if (propType == null) if (propType == null)
@ -78,6 +82,7 @@ namespace RobloxFiles.XmlFormat
case PropertyType.String: case PropertyType.String:
propType = (prop.HasRawBuffer ? "BinaryString" : "string"); propType = (prop.HasRawBuffer ? "BinaryString" : "string");
break; break;
default: break;
} }
} }
@ -89,6 +94,19 @@ namespace RobloxFiles.XmlFormat
return null; return null;
} }
if (prop.Type == PropertyType.SharedString)
{
SharedString value = prop.CastValue<SharedString>();
if (value.ComputedKey == null)
{
var newShared = SharedString.FromBuffer(value.SharedValue);
value.Key = newShared.ComputedKey;
}
file.SharedStrings.Add(value.Key);
}
XmlElement propElement = doc.CreateElement(propType); XmlElement propElement = doc.CreateElement(propType);
propElement.SetAttribute("name", prop.Name); propElement.SetAttribute("name", prop.Name);
@ -102,12 +120,6 @@ namespace RobloxFiles.XmlFormat
propNode = newNode; propNode = newNode;
} }
if (prop.Type == PropertyType.SharedString)
{
SharedString value = prop.CastValue<SharedString>();
file.SharedStrings.Add(value.MD5_Key);
}
return propNode; return propNode;
} }
@ -125,10 +137,18 @@ namespace RobloxFiles.XmlFormat
var props = instance.RefreshProperties(); var props = instance.RefreshProperties();
foreach (string propName in props.Keys) var orderedKeys = props
.OrderBy(pair => pair.Key)
.Select(pair => pair.Key);
foreach (string propName in orderedKeys)
{ {
Property prop = props[propName]; Property prop = props[propName];
XmlNode propNode = WriteProperty(prop, doc, file); XmlNode propNode = WriteProperty(prop, doc, file);
if (propNode == null)
continue;
propsNode.AppendChild(propNode); propsNode.AppendChild(propNode);
} }
@ -151,12 +171,12 @@ namespace RobloxFiles.XmlFormat
var binaryWriter = XmlPropertyTokens.GetHandler<BinaryStringToken>(); var binaryWriter = XmlPropertyTokens.GetHandler<BinaryStringToken>();
var binaryBuffer = new Property("SharedString", PropertyType.String); var binaryBuffer = new Property("SharedString", PropertyType.String);
foreach (string md5 in file.SharedStrings) foreach (string key in file.SharedStrings)
{ {
XmlElement sharedString = doc.CreateElement("SharedString"); XmlElement sharedString = doc.CreateElement("SharedString");
sharedString.SetAttribute("md5", md5); sharedString.SetAttribute("md5", key);
binaryBuffer.RawBuffer = SharedString.FindRecord(md5); binaryBuffer.RawBuffer = SharedString.Find(key);
binaryWriter.WriteProperty(binaryBuffer, doc, sharedString); binaryWriter.WriteProperty(binaryBuffer, doc, sharedString);
sharedStrings.AppendChild(sharedString); sharedStrings.AppendChild(sharedString);

23
XmlFormat/Tokens/ProtectedString.cs
View File

@ -1,4 +1,5 @@
using System.Xml; using System.Text;
using System.Xml;
using RobloxFiles.DataTypes; using RobloxFiles.DataTypes;
namespace RobloxFiles.XmlFormat.PropertyTokens namespace RobloxFiles.XmlFormat.PropertyTokens
@ -10,7 +11,7 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
public bool ReadProperty(Property prop, XmlNode token) public bool ReadProperty(Property prop, XmlNode token)
{ {
ProtectedString contents = token.InnerText; ProtectedString contents = token.InnerText;
prop.Type = PropertyType.String; prop.Type = PropertyType.ProtectedString;
prop.Value = contents; prop.Value = contents;
return true; return true;
@ -18,16 +19,26 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
public void WriteProperty(Property prop, XmlDocument doc, XmlNode node) public void WriteProperty(Property prop, XmlDocument doc, XmlNode node)
{ {
string value = prop.CastValue<ProtectedString>(); ProtectedString value = prop.CastValue<ProtectedString>();
if (value.Contains("\r") || value.Contains("\n")) if (value.IsCompiled)
{ {
XmlCDataSection cdata = doc.CreateCDataSection(value); var binary = XmlPropertyTokens.GetHandler<BinaryStringToken>();
binary.WriteProperty(prop, doc, node);
}
else
{
string contents = Encoding.UTF8.GetString(value.RawBuffer);
if (contents.Contains("\r") || contents.Contains("\n"))
{
XmlCDataSection cdata = doc.CreateCDataSection(contents);
node.AppendChild(cdata); node.AppendChild(cdata);
} }
else else
{ {
node.InnerText = value; node.InnerText = contents;
}
} }
} }
} }

16
XmlFormat/Tokens/SharedString.cs
View File

@ -9,17 +9,25 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
public bool ReadProperty(Property prop, XmlNode token) public bool ReadProperty(Property prop, XmlNode token)
{ {
string md5 = token.InnerText; string key = token.InnerText;
prop.Type = PropertyType.SharedString; prop.Type = PropertyType.SharedString;
prop.Value = new SharedString(md5); prop.Value = new SharedString(key);
return true; return true;
} }
public void WriteProperty(Property prop, XmlDocument doc, XmlNode node) public void WriteProperty(Property prop, XmlDocument doc, XmlNode node)
{ {
SharedString value = prop.CastValue<SharedString>(); var value = prop.CastValue<SharedString>();
node.InnerText = value.MD5_Key; string key = value.Key;
if (value.ComputedKey == null)
{
var newShared = SharedString.FromBuffer(value.SharedValue);
key = newShared.ComputedKey;
}
node.InnerText = key;
} }
} }
} }

2
XmlFormat/XmlRobloxFile.cs
View File

@ -103,7 +103,7 @@ namespace RobloxFiles
foreach (Property sharedProp in sharedProps) foreach (Property sharedProp in sharedProps)
{ {
SharedString shared = sharedProp.CastValue<SharedString>(); SharedString shared = sharedProp.CastValue<SharedString>();
SharedStrings.Add(shared.MD5_Key); SharedStrings.Add(shared.Key);
} }
} }
else else

6
packages.config
View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Costura.Fody" version="3.3.3" targetFramework="net452" /> <package id="Costura.Fody" version="4.1.0" targetFramework="net472" />
<package id="Fody" version="4.2.1" targetFramework="net452" developmentDependency="true" /> <package id="Fody" version="6.2.0" targetFramework="net472" developmentDependency="true" />
<package id="Konscious.Security.Cryptography.Blake2" version="1.0.9" targetFramework="net472" />
<package id="lz4net" version="1.0.15.93" targetFramework="net452" /> <package id="lz4net" version="1.0.15.93" targetFramework="net452" />
<package id="System.Numerics.Vectors" version="4.5.0" targetFramework="net472" />
</packages> </packages>