Brought spec up to date, improvements to stability

This commit is contained in:
CloneTrooper1019 2020-07-12 20:19:30 -05:00
parent 7359b6efb7
commit 57fd3f8a25
29 changed files with 551 additions and 245 deletions

View File

@ -15,20 +15,22 @@ namespace RobloxFiles
// Header Specific
public const string MagicHeader = "<roblox!\x89\xff\x0d\x0a\x1a\x0a";
public ushort Version;
public uint NumClasses;
public uint NumInstances;
public long Reserved;
public ushort Version { get; internal set; }
public uint NumClasses { get; internal set; }
public uint NumInstances { get; internal set; }
public long Reserved { get; internal set; }
// Runtime Specific
public List<BinaryRobloxFileChunk> Chunks = new List<BinaryRobloxFileChunk>();
internal List<BinaryRobloxFileChunk> ChunksImpl = new List<BinaryRobloxFileChunk>();
public IReadOnlyList<BinaryRobloxFileChunk> Chunks => ChunksImpl;
public override string ToString() => GetType().Name;
public Instance[] Instances;
public INST[] Classes;
public Instance[] Instances { get; internal set; }
public INST[] Classes { get; internal set; }
internal META META = null;
internal SSTR SSTR = null;
internal SIGN SIGN = null;
public bool HasMetadata => (META != null);
public Dictionary<string, string> Metadata => META?.Data;
@ -36,6 +38,9 @@ namespace RobloxFiles
public bool HasSharedStrings => (SSTR != null);
public IReadOnlyDictionary<uint, SharedString> SharedStrings => SSTR?.Strings;
public bool HasSignatures => (SIGN != null);
public IReadOnlyList<Signature> Signatures => SIGN?.Signatures;
public BinaryRobloxFile()
{
Name = "BinaryRobloxFile";
@ -71,12 +76,10 @@ namespace RobloxFiles
{
try
{
BinaryRobloxFileChunk chunk = new BinaryRobloxFileChunk(reader);
string chunkType = chunk.ChunkType;
var chunk = new BinaryRobloxFileChunk(reader);
IBinaryFileChunk handler = null;
switch (chunkType)
switch (chunk.ChunkType)
{
case "INST":
handler = new INST();
@ -93,22 +96,27 @@ namespace RobloxFiles
case "SSTR":
handler = new SSTR();
break;
case "SIGN":
handler = new SIGN();
break;
case "END\0":
Chunks.Add(chunk);
ChunksImpl.Add(chunk);
reading = false;
break;
default:
Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", chunkType);
case string unhandled:
Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", unhandled);
break;
default: break;
}
if (handler != null)
{
using (BinaryRobloxFileReader dataReader = chunk.GetDataReader(this))
handler.LoadFromReader(dataReader);
chunk.Handler = handler;
Chunks.Add(chunk);
using (var dataReader = chunk.GetDataReader(this))
handler.Load(dataReader);
ChunksImpl.Add(chunk);
}
}
catch (EndOfStreamException)
@ -129,7 +137,7 @@ namespace RobloxFiles
{
// Clear the existing data.
Referent = "-1";
Chunks.Clear();
ChunksImpl.Clear();
NumInstances = 0;
NumClasses = 0;
@ -148,7 +156,7 @@ namespace RobloxFiles
// Write the PROP chunks.
foreach (INST inst in Classes)
{
Dictionary<string, PROP> props = PROP.CollectProperties(writer, inst);
var props = PROP.CollectProperties(writer, inst);
foreach (string propName in props.Keys)
{
@ -158,26 +166,23 @@ namespace RobloxFiles
}
// Write the PRNT chunk.
PRNT parents = new PRNT();
var parents = new PRNT();
writer.SaveChunk(parents);
// Write the SSTR chunk.
if (HasSharedStrings)
{
var sharedStrings = SSTR.SaveAsChunk(writer);
Chunks.Insert(0, sharedStrings);
}
writer.SaveChunk(SSTR, 0);
// Write the META chunk.
if (HasMetadata)
{
var metaChunk = META.SaveAsChunk(writer);
Chunks.Insert(0, metaChunk);
}
writer.SaveChunk(META, 0);
// Write the END_ chunk.
var endChunk = writer.WriteEndChunk();
Chunks.Add(endChunk);
// Write the SIGN chunk.
if (HasSignatures)
writer.SaveChunk(SIGN);
// Write the END chunk.
writer.WriteChunk("END", "</roblox>");
}
//////////////////////////////////////////////////////////////////////////

View File

@ -16,7 +16,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
public override string ToString() => ClassName;
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
@ -59,10 +59,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
file.Classes[ClassIndex] = this;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
writer.Write(ClassIndex);
writer.WriteString(ClassName);
@ -72,7 +70,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
if (IsService)
{
BinaryRobloxFile file = writer.File;
var file = writer.File;
foreach (int instId in InstanceIds)
{
@ -80,8 +78,6 @@ namespace RobloxFiles.BinaryFormat.Chunks
writer.Write(service.Parent == file);
}
}
return writer.FinishWritingChunk();
}
}
}

View File

@ -6,7 +6,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
{
public Dictionary<string, string> Data = new Dictionary<string, string>();
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
int numEntries = reader.ReadInt32();
@ -21,18 +21,15 @@ namespace RobloxFiles.BinaryFormat.Chunks
file.META = this;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
writer.Write(Data.Count);
foreach (var kvPair in Data)
foreach (var pair in Data)
{
writer.WriteString(kvPair.Key);
writer.WriteString(kvPair.Value);
writer.WriteString(pair.Key);
writer.WriteString(pair.Value);
}
return writer.FinishWritingChunk();
}
}
}

View File

@ -1,29 +1,29 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
namespace RobloxFiles.BinaryFormat.Chunks
{
public class PRNT : IBinaryFileChunk
{
public byte Format { get; private set; }
public int NumRelations { get; private set; }
private const byte FORMAT = 0;
public List<int> ChildrenIds { get; private set; }
public List<int> ParentIds { get; private set; }
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
Format = reader.ReadByte();
NumRelations = reader.ReadInt32();
byte format = reader.ReadByte();
int idCount = reader.ReadInt32();
ChildrenIds = reader.ReadInstanceIds(NumRelations);
ParentIds = reader.ReadInstanceIds(NumRelations);
if (format != FORMAT)
throw new Exception($"Unexpected PRNT format: {format} (expected {FORMAT}!)");
for (int i = 0; i < NumRelations; i++)
var childIds = reader.ReadInstanceIds(idCount);
var parentIds = reader.ReadInstanceIds(idCount);
for (int i = 0; i < idCount; i++)
{
int childId = ChildrenIds[i];
int parentId = ParentIds[i];
int childId = childIds[i];
int parentId = parentIds[i];
Instance child = file.Instances[childId];
child.Parent = (parentId >= 0 ? file.Instances[parentId] : file);
@ -31,15 +31,13 @@ namespace RobloxFiles.BinaryFormat.Chunks
}
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
var postInstances = writer.PostInstances;
var idCount = postInstances.Count;
Format = 0;
NumRelations = 0;
ChildrenIds = new List<int>();
ParentIds = new List<int>();
var childIds = new List<int>();
var parentIds = new List<int>();
foreach (Instance inst in writer.PostInstances)
{
@ -51,19 +49,15 @@ namespace RobloxFiles.BinaryFormat.Chunks
if (parent != null)
parentId = int.Parse(parent.Referent);
ChildrenIds.Add(childId);
ParentIds.Add(parentId);
NumRelations++;
childIds.Add(childId);
parentIds.Add(parentId);
}
writer.Write(Format);
writer.Write(NumRelations);
writer.Write(FORMAT);
writer.Write(idCount);
writer.WriteInstanceIds(ChildrenIds);
writer.WriteInstanceIds(ParentIds);
return writer.FinishWritingChunk();
writer.WriteInstanceIds(childIds);
writer.WriteInstanceIds(parentIds);
}
}
}

View File

@ -29,7 +29,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
return $"{Type} {ClassName}.{Name}";
}
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
@ -71,7 +71,6 @@ namespace RobloxFiles.BinaryFormat.Chunks
}
});
// Read the property data based on the property type.
switch (Type)
{
case PropertyType.String:
@ -500,13 +499,22 @@ namespace RobloxFiles.BinaryFormat.Chunks
readProperties(i =>
{
uint key = SharedKeys[i];
return file.SharedStrings[key];
});
break;
case PropertyType.ProtectedString:
readProperties(i =>
{
int length = reader.ReadInt32();
byte[] buffer = reader.ReadBytes(length);
return new ProtectedString(buffer);
});
break;
default:
Console.WriteLine("Unhandled property type: {0}!", Type);
Console.Error.WriteLine("Unhandled property type: {0}!", Type);
break;
//
}
@ -546,7 +554,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
return propMap;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
BinaryRobloxFile file = writer.File;
@ -567,9 +575,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.Add(prop);
}
writer.StartWritingChunk(this);
writer.Write(ClassIndex);
writer.WriteString(Name);
writer.Write(TypeId);
@ -996,12 +1002,12 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.ForEach(prop =>
{
SharedString shared = prop.CastValue<SharedString>();
string key = shared.MD5_Key;
var shared = prop.CastValue<SharedString>();
string key = shared.Key;
if (!sstr.Lookup.ContainsKey(key))
{
uint id = (uint)(sstr.NumHashes++);
uint id = (uint)(sstr.Lookup.Count);
sstr.Strings.Add(id, shared);
sstr.Lookup.Add(key, id);
}
@ -1012,10 +1018,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
writer.WriteInterleaved(sharedKeys);
break;
//
}
case PropertyType.ProtectedString:
props.ForEach(prop =>
{
var protect = prop.CastValue<ProtectedString>();
byte[] buffer = protect.RawBuffer;
return writer.FinishWritingChunk();
writer.Write(buffer.Length);
writer.Write(buffer);
});
break;
default: break;
}
}
}
}

View File

@ -0,0 +1,56 @@
namespace RobloxFiles.BinaryFormat.Chunks
{
public struct Signature
{
public int Version;
public long Id;
public int Length;
public byte[] Data;
}
public class SIGN : IBinaryFileChunk
{
public Signature[] Signatures;
public void Load(BinaryRobloxFileReader reader)
{
int numSignatures = reader.ReadInt32();
Signatures = new Signature[numSignatures];
for (int i = 0; i < numSignatures; i++)
{
var signature = new Signature
{
Version = reader.ReadInt32(),
Id = reader.ReadInt64(),
Length = reader.ReadInt32(),
};
signature.Data = reader.ReadBytes(signature.Length);
Signatures[i] = signature;
}
var file = reader.File;
file.SIGN = this;
}
public void Save(BinaryRobloxFileWriter writer)
{
writer.Write(Signatures.Length);
for (int i = 0; i < Signatures.Length; i++)
{
var signature = Signatures[i];
signature.Length = signature.Data.Length;
writer.Write(signature.Version);
writer.Write(signature.Id);
writer.Write(signature.Length);
writer.Write(signature.Data);
}
}
}
}

View File

@ -6,56 +6,54 @@ namespace RobloxFiles.BinaryFormat.Chunks
{
public class SSTR : IBinaryFileChunk
{
public int Version;
public int NumHashes;
private const int FORMAT = 0;
public Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
public Dictionary<uint, SharedString> Strings = new Dictionary<uint, SharedString>();
internal Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
internal Dictionary<uint, SharedString> Strings = new Dictionary<uint, SharedString>();
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
Version = reader.ReadInt32();
NumHashes = reader.ReadInt32();
int format = reader.ReadInt32();
int numHashes = reader.ReadInt32();
for (uint id = 0; id < NumHashes; id++)
if (format != FORMAT)
throw new Exception($"Unexpected SSTR format: {format} (expected {FORMAT}!)");
for (uint id = 0; id < numHashes; id++)
{
byte[] md5 = reader.ReadBytes(16);
int length = reader.ReadInt32();
byte[] data = reader.ReadBytes(length);
byte[] hash = reader.ReadBytes(16);
string key = Convert.ToBase64String(hash);
byte[] data = reader.ReadBuffer();
SharedString value = SharedString.FromBuffer(data);
Lookup.Add(value.MD5_Key, id);
Lookup.Add(key, id);
Strings.Add(id, value);
}
file.SSTR = this;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
writer.Write(Version);
writer.Write(NumHashes);
writer.Write(FORMAT);
writer.Write(Lookup.Count);
foreach (var pair in Lookup)
{
string key = pair.Key;
byte[] md5 = Convert.FromBase64String(key);
writer.Write(md5);
byte[] hash = Convert.FromBase64String(key);
writer.Write(hash);
SharedString value = Strings[pair.Value];
byte[] buffer = SharedString.FindRecord(value.MD5_Key);
byte[] buffer = SharedString.Find(value.Key);
writer.Write(buffer.Length);
writer.Write(buffer);
}
return writer.FinishWritingChunk();
}
}
}

View File

@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
@ -18,14 +17,16 @@ namespace RobloxFiles.BinaryFormat
public string ChunkType { get; private set; }
public long ChunkStart { get; private set; }
public Dictionary<string, INST> ClassMap;
public readonly BinaryRobloxFile File;
// Dictionary mapping ClassNames to their INST chunks.
private readonly Dictionary<string, INST> ClassMap;
// Instances in parent->child order
public List<Instance> Instances;
private readonly List<Instance> Instances;
// Instances in child->parent order
public List<Instance> PostInstances;
internal List<Instance> PostInstances { get; private set; }
public BinaryRobloxFileWriter(BinaryRobloxFile file, Stream workBuffer = null) : base(workBuffer ?? new MemoryStream())
{
@ -104,13 +105,13 @@ namespace RobloxFiles.BinaryFormat
}
// Encodes an int for an interleaved buffer.
private int EncodeInt(int value)
private static int EncodeInt(int value)
{
return (value << 1) ^ (value >> 31);
}
// Encodes a float for an interleaved buffer.
private float EncodeFloat(float value)
private static float EncodeFloat(float value)
{
byte[] buffer = BitConverter.GetBytes(value);
uint bits = BitConverter.ToUInt32(buffer, 0);
@ -171,9 +172,9 @@ namespace RobloxFiles.BinaryFormat
Instances.Add(instance);
string className = instance.ClassName;
INST inst = null;
INST inst;
if (!ClassMap.ContainsKey(className))
if (!ClassMap.TryGetValue(className, out inst))
{
inst = new INST()
{
@ -184,10 +185,6 @@ namespace RobloxFiles.BinaryFormat
ClassMap.Add(className, inst);
}
else
{
inst = ClassMap[className];
}
inst.NumInstances++;
inst.InstanceIds.Add(instId);
@ -222,7 +219,7 @@ namespace RobloxFiles.BinaryFormat
}
// Marks that we are writing a chunk.
public bool StartWritingChunk(string chunkType)
private bool StartWritingChunk(string chunkType)
{
if (chunkType.Length != 4)
throw new Exception("BinaryFileWriter.StartWritingChunk - ChunkType length should be 4!");
@ -241,7 +238,7 @@ namespace RobloxFiles.BinaryFormat
}
// Marks that we are writing a chunk.
public bool StartWritingChunk(IBinaryFileChunk chunk)
private bool StartWritingChunk(IBinaryFileChunk chunk)
{
if (!WritingChunk)
{
@ -257,7 +254,7 @@ namespace RobloxFiles.BinaryFormat
}
// Compresses the data that was written into a BinaryRobloxFileChunk and writes it.
public BinaryRobloxFileChunk FinishWritingChunk(bool compress = true)
private BinaryRobloxFileChunk FinishWritingChunk(bool compress = true)
{
if (!WritingChunk)
throw new Exception($"BinaryRobloxFileWriter: Cannot finish writing a chunk without starting one!");
@ -283,18 +280,36 @@ namespace RobloxFiles.BinaryFormat
return chunk;
}
public void SaveChunk(IBinaryFileChunk handler)
internal BinaryRobloxFileChunk SaveChunk(IBinaryFileChunk handler, int insertPos = -1)
{
var chunk = handler.SaveAsChunk(this);
File.Chunks.Add(chunk);
StartWritingChunk(handler);
handler.Save(this);
var chunk = FinishWritingChunk();
if (insertPos >= 0)
File.ChunksImpl.Insert(insertPos, chunk);
else
File.ChunksImpl.Add(chunk);
return chunk;
}
public BinaryRobloxFileChunk WriteEndChunk()
internal BinaryRobloxFileChunk WriteChunk(string chunkType, string content, bool compress = false)
{
StartWritingChunk("END\0");
WriteString("</roblox>", true);
if (chunkType.Length > 4)
chunkType = chunkType.Substring(0, 4);
return FinishWritingChunk(false);
while (chunkType.Length < 4)
chunkType += '\0';
StartWritingChunk(chunkType);
WriteString(content);
var chunk = FinishWritingChunk(compress);
File.ChunksImpl.Add(chunk);
return chunk;
}
}
}

View File

@ -1,27 +1,54 @@
namespace RobloxFiles.DataTypes
using System.Text;
namespace RobloxFiles.DataTypes
{
/// <summary>
/// ProtectedString is a type used by some of the XML properties.
/// Here, it only exists as a wrapper class for strings.
/// ProtectedString is a type used by the Source property of scripts.
/// If constructed as an array of bytes, it's assumed to be compiled byte-code.
/// </summary>
public class ProtectedString
{
public readonly string ProtectedValue;
public override string ToString() => ProtectedValue;
public readonly bool IsCompiled;
public readonly byte[] RawBuffer;
public override string ToString()
{
if (IsCompiled)
return $"byte[{RawBuffer.Length}]";
return Encoding.UTF8.GetString(RawBuffer);
}
public ProtectedString(string value)
{
ProtectedValue = value;
IsCompiled = false;
RawBuffer = Encoding.UTF8.GetBytes(value);
}
public ProtectedString(byte[] compiled)
{
IsCompiled = true;
RawBuffer = compiled;
}
public static implicit operator string(ProtectedString protectedString)
{
return protectedString.ProtectedValue;
return Encoding.UTF8.GetString(protectedString.RawBuffer);
}
public static implicit operator ProtectedString(string value)
{
return new ProtectedString(value);
}
public static implicit operator byte[](ProtectedString protectedString)
{
return protectedString.RawBuffer;
}
public static implicit operator ProtectedString(byte[] value)
{
return new ProtectedString(value);
}
}
}

View File

@ -1,43 +1,58 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Security.Cryptography;
using System.Collections.Generic;
using Konscious.Security.Cryptography;
namespace RobloxFiles.DataTypes
{
// SharedString is a datatype that takes a sequence of bytes and stores it in a
// lookup table that is shared by the entire file. It originally used MD5 for the
// hashing, but Roblox now uses Blake2B to avoid the obvious problems with using MD5.
// In practice the value of a SharedString does not have to match the hash of the
// data it represents, it just needs to be distinct and MUST be 16 bytes long.
// The XML format still uses 'md5' as its attribute key to the lookup table.
public class SharedString
{
private static Dictionary<string, byte[]> Records = new Dictionary<string, byte[]>();
public readonly string MD5_Key;
private static Dictionary<string, byte[]> Lookup = new Dictionary<string, byte[]>();
public string Key { get; internal set; }
public string ComputedKey { get; internal set; }
public byte[] SharedValue => FindRecord(MD5_Key);
public override string ToString() => $"MD5 Key: {MD5_Key}";
public byte[] SharedValue => Find(ComputedKey ?? Key);
public override string ToString() => $"Key: {ComputedKey ?? Key}";
internal SharedString(string md5)
internal SharedString(string key)
{
MD5_Key = md5;
Key = key;
}
internal static void Register(string key, byte[] buffer)
{
Lookup.Add(key, buffer);
}
private SharedString(byte[] buffer)
{
using (MD5 md5 = MD5.Create())
using (HMACBlake2B blake2B = new HMACBlake2B(16 * 8))
{
byte[] hash = md5.ComputeHash(buffer);
MD5_Key = Convert.ToBase64String(hash);
byte[] hash = blake2B.ComputeHash(buffer);
ComputedKey = Convert.ToBase64String(hash);
Key = ComputedKey;
}
if (Records.ContainsKey(MD5_Key))
if (Lookup.ContainsKey(ComputedKey))
return;
Records.Add(MD5_Key, buffer);
Register(ComputedKey, buffer);
}
public static byte[] FindRecord(string key)
public static byte[] Find(string key)
{
byte[] result = null;
if (Records.ContainsKey(key))
result = Records[key];
if (Lookup.ContainsKey(key))
result = Lookup[key];
return result;
}

View File

@ -2,7 +2,7 @@
{
public interface IBinaryFileChunk
{
void LoadFromReader(BinaryRobloxFileReader reader);
BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer);
void Load(BinaryRobloxFileReader reader);
void Save(BinaryRobloxFileWriter writer);
}
}

View File

@ -6,17 +6,23 @@ using System.Threading.Tasks;
namespace RobloxFiles
{
/// <summary>
/// Represents a loaded *.rbxl/*.rbxm Roblox file.
/// The contents of the RobloxFile are stored as its children.
/// Represents a loaded Roblox place/model file.
/// RobloxFile is an Instance and its children are the contents of the file.
/// </summary>
public abstract class RobloxFile : Instance
{
protected abstract void ReadFile(byte[] buffer);
/// <summary>
/// Saves this RobloxFile to the provided stream.
/// </summary>
/// <param name="stream">The stream to save to.</param>
public abstract void Save(Stream stream);
/// <summary>
/// Opens a RobloxFile using the provided buffer.
/// </summary>
/// <returns>The opened RobloxFile.</returns>
public static RobloxFile Open(byte[] buffer)
{
if (buffer.Length > 14)
@ -43,6 +49,7 @@ namespace RobloxFiles
/// Opens a Roblox file by reading from a provided Stream.
/// </summary>
/// <param name="stream">The stream to read the Roblox file from.</param>
/// <returns>The opened RobloxFile.</returns>
public static RobloxFile Open(Stream stream)
{
byte[] buffer;
@ -60,6 +67,7 @@ namespace RobloxFiles
/// Opens a Roblox file from a provided file path.
/// </summary>
/// <param name="filePath">A path to a Roblox file to be opened.</param>
/// <returns>The opened RobloxFile.</returns>
public static RobloxFile Open(string filePath)
{
byte[] buffer = File.ReadAllBytes(filePath);
@ -70,6 +78,7 @@ namespace RobloxFiles
/// Creates and runs a Task to open a Roblox file from a byte sequence that represents the file.
/// </summary>
/// <param name="buffer">A byte sequence that represents the file.</param>
/// <returns>A task which will complete once the file is opened with the resulting RobloxFile.</returns>
public static Task<RobloxFile> OpenAsync(byte[] buffer)
{
return Task.Run(() => Open(buffer));
@ -79,6 +88,7 @@ namespace RobloxFiles
/// Creates and runs a Task to open a Roblox file using a provided Stream.
/// </summary>
/// <param name="stream">The stream to read the Roblox file from.</param>
/// <returns>A task which will complete once the file is opened with the resulting RobloxFile.</returns>
public static Task<RobloxFile> OpenAsync(Stream stream)
{
return Task.Run(() => Open(stream));
@ -88,9 +98,42 @@ namespace RobloxFiles
/// Opens a Roblox file from a provided file path.
/// </summary>
/// <param name="filePath">A path to a Roblox file to be opened.</param>
/// <returns>A task which will complete once the file is opened with the resulting RobloxFile.</returns>
public static Task<RobloxFile> OpenAsync(string filePath)
{
return Task.Run(() => Open(filePath));
}
/// <summary>
/// Saves this RobloxFile to the provided file path.
/// </summary>
/// <param name="filePath">A path to where the file should be saved.</param>
public void Save(string filePath)
{
using (FileStream stream = File.OpenWrite(filePath))
{
Save(stream);
}
}
/// <summary>
/// Asynchronously saves this RobloxFile to the provided stream.
/// </summary>
/// <param name="stream">The stream to save to.</param>
/// <returns>A task which will complete upon the save's completion.</returns>
public Task SaveAsync(Stream stream)
{
return Task.Run(() => Save(stream));
}
/// <summary>
/// Asynchronously saves this RobloxFile to the provided file path.
/// </summary>
/// <param name="filePath">A path to where the file should be saved.</param>
/// <returns>A task which will complete upon the save's completion.</returns>
public Task SaveAsync(string filePath)
{
return Task.Run(() => Save(filePath));
}
}
}

View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="packages\Costura.Fody.3.3.3\build\Costura.Fody.props" Condition="Exists('packages\Costura.Fody.3.3.3\build\Costura.Fody.props')" />
<Import Project="packages\Costura.Fody.4.1.0\build\Costura.Fody.props" Condition="Exists('packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -52,9 +52,11 @@
<StartupObject />
</PropertyGroup>
<ItemGroup>
<Reference Include="Costura, Version=3.3.3.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
<HintPath>packages\Costura.Fody.3.3.3\lib\net40\Costura.dll</HintPath>
<Private>True</Private>
<Reference Include="Costura, Version=4.1.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
<HintPath>packages\Costura.Fody.4.1.0\lib\net40\Costura.dll</HintPath>
</Reference>
<Reference Include="Konscious.Security.Cryptography.Blake2, Version=1.0.9.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>packages\Konscious.Security.Cryptography.Blake2.1.0.9\lib\net46\Konscious.Security.Cryptography.Blake2.dll</HintPath>
</Reference>
<Reference Include="LZ4, Version=1.0.15.93, Culture=neutral, PublicKeyToken=62e1b5ec1eec9bdd, processorArchitecture=MSIL">
<HintPath>packages\lz4net.1.0.15.93\lib\net4-client\LZ4.dll</HintPath>
@ -62,6 +64,10 @@
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Numerics" />
<Reference Include="System.Numerics.Vectors, Version=4.1.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
</Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
@ -76,6 +82,7 @@
<Compile Include="BinaryFormat\Chunks\META.cs" />
<Compile Include="BinaryFormat\Chunks\PRNT.cs" />
<Compile Include="BinaryFormat\Chunks\PROP.cs" />
<Compile Include="BinaryFormat\Chunks\SIGN.cs" />
<Compile Include="BinaryFormat\Chunks\SSTR.cs" />
<Compile Include="BinaryFormat\IO\BinaryFileReader.cs" />
<Compile Include="BinaryFormat\IO\BinaryFileWriter.cs" />
@ -88,7 +95,6 @@
<Compile Include="Interfaces\IBinaryFileChunk.cs" />
<Compile Include="Generated\Classes.cs" />
<Compile Include="Generated\Enums.cs" />
<Compile Include="LibTest\Program.cs" />
<Compile Include="Tree\Attributes.cs" />
<Compile Include="Tree\Property.cs" />
<Compile Include="Tree\Instance.cs" />
@ -167,24 +173,18 @@
</BootstrapperPackage>
</ItemGroup>
<ItemGroup>
<None Include="LibTest\Binary.rbxl">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Include="LibTest\Xml.rbxlx">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<PropertyGroup>
<PostBuildEvent>copy /y $(TargetPath) $(ProjectDir)$(TargetFileName)</PostBuildEvent>
</PropertyGroup>
<Import Project="packages\Fody.4.2.1\build\Fody.targets" Condition="Exists('packages\Fody.4.2.1\build\Fody.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('packages\Fody.4.2.1\build\Fody.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Fody.4.2.1\build\Fody.targets'))" />
<Error Condition="!Exists('packages\Costura.Fody.3.3.3\build\Costura.Fody.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Costura.Fody.3.3.3\build\Costura.Fody.props'))" />
<Error Condition="!Exists('packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Costura.Fody.4.1.0\build\Costura.Fody.props'))" />
<Error Condition="!Exists('packages\Fody.6.2.0\build\Fody.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Fody.6.2.0\build\Fody.targets'))" />
</Target>
<Import Project="packages\Fody.6.2.0\build\Fody.targets" Condition="Exists('packages\Fody.6.2.0\build\Fody.targets')" />
</Project>

Binary file not shown.

View File

@ -5,6 +5,8 @@ VisualStudioVersion = 16.0.29920.165
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobloxFileFormat", "RobloxFileFormat.csproj", "{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RobloxFileFormat.UnitTest", "UnitTest\RobloxFileFormat.UnitTest.csproj", "{6BCA31B2-58D8-4689-9929-88E16040BF29}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -15,6 +17,10 @@ Global
{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CF50C0E2-23A7-4DC1-B4B2-E60CDE716253}.Release|Any CPU.Build.0 = Release|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6BCA31B2-58D8-4689-9929-88E16040BF29}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

View File

@ -54,7 +54,7 @@ namespace RobloxFiles
}
internal BinaryReader reader;
internal BinaryWriter writer;
// internal BinaryWriter writer;
internal int readInt() => reader.ReadInt32();
internal byte readByte() => reader.ReadByte();

View File

@ -521,10 +521,10 @@ namespace RobloxFiles
PropertyType propType = PropertyType.Unknown;
if (fieldType.IsEnum)
propType = PropertyType.Enum;
else if (Property.Types.ContainsKey(fieldType))
if (Property.Types.ContainsKey(fieldType))
propType = Property.Types[fieldType];
else if (fieldType.IsEnum)
propType = PropertyType.Enum;
if (propType != PropertyType.Unknown)
{

View File

@ -42,6 +42,7 @@ namespace RobloxFiles
Color3uint8,
Int64,
SharedString,
ProtectedString
}
public class Property
@ -60,6 +61,7 @@ namespace RobloxFiles
internal static BindingFlags BindingFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.FlattenHierarchy | BindingFlags.IgnoreCase;
internal static MemberTypes FieldOrProperty = MemberTypes.Field | MemberTypes.Property;
// !! FIXME: Map typeof(ProtectedString) to PropertyType.ProtectedString when binary files are allowed to read it.
public static readonly IReadOnlyDictionary<Type, PropertyType> Types = new Dictionary<Type, PropertyType>()
{
{ typeof(Axes), PropertyType.Axes },
@ -98,17 +100,23 @@ namespace RobloxFiles
private void ImproviseRawBuffer()
{
if (RawValue is byte[])
{
RawBuffer = RawValue as byte[];
return;
}
else if (RawValue is SharedString)
if (RawValue is SharedString)
{
var sharedString = CastValue<SharedString>();
RawBuffer = sharedString.SharedValue;
return;
}
else if (RawValue is ProtectedString)
{
var protectedString = CastValue<ProtectedString>();
RawBuffer = protectedString.RawBuffer;
return;
}
else if (RawValue is byte[])
{
RawBuffer = RawValue as byte[];
return;
}
switch (Type)
{
@ -127,7 +135,7 @@ namespace RobloxFiles
case PropertyType.Double:
RawBuffer = BitConverter.GetBytes((double)Value);
break;
//
default: break;
}
}
@ -142,7 +150,11 @@ namespace RobloxFiles
if (typeName == Name)
{
FieldInfo directField = instType.GetField(typeName, BindingFlags.DeclaredOnly);
FieldInfo directField = instType
.GetFields()
.Where(field => field.Name.StartsWith(Name))
.Where(field => field.DeclaringType == instType)
.FirstOrDefault();
if (directField != null)
{

View File

@ -2,19 +2,59 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using RobloxFiles.DataTypes;
namespace RobloxFiles
namespace RobloxFiles.UnitTest
{
// If the solution is built as an exe, this class is
// used to drive some basic testing of the library.
internal static class Program
static class Program
{
const string pattern = "\\d+$";
static void PrintTreeImpl(Instance inst, int stack = 0)
{
string padding = "";
string extension = "";
for (int i = 0; i < stack; i++)
padding += '\t';
switch (inst.ClassName)
{
case "Script":
extension = ".server.lua";
break;
case "LocalScript":
extension = ".client.lua";
break;
case "ModuleScript":
extension = ".lua";
break;
default: break;
}
Console.WriteLine($"{padding}{inst.Name}{extension}");
var children = inst
.GetChildren()
.ToList();
children.ForEach(child => PrintTreeImpl(child, stack + 1));
}
static void PrintTree(string path)
{
Console.WriteLine("Opening file...");
RobloxFile target = RobloxFile.Open(path);
foreach (Instance child in target.GetChildren())
PrintTreeImpl(child);
Debugger.Break();
}
static void CountAssets(string path)
{
Console.WriteLine("Opening file...");
@ -23,6 +63,14 @@ namespace RobloxFiles
var workspace = target.FindFirstChildOfClass<Workspace>();
var assets = new HashSet<string>();
if (workspace == null)
{
Console.WriteLine("No workspace found!");
Debugger.Break();
return;
}
foreach (Instance inst in workspace.GetDescendants())
{
var instPath = inst.GetFullName();
@ -63,21 +111,18 @@ namespace RobloxFiles
if (args.Length > 0)
{
string path = args[0];
CountAssets(path);
PrintTree(path);
}
else
{
RobloxFile bin = RobloxFile.Open(@"LibTest\Binary.rbxl");
RobloxFile xml = RobloxFile.Open(@"LibTest\Xml.rbxlx");
RobloxFile bin = RobloxFile.Open(@"Files\Binary.rbxl");
RobloxFile xml = RobloxFile.Open(@"Files\Xml.rbxlx");
Console.WriteLine("Files opened! Pausing execution for debugger analysis...");
Debugger.Break();
using (FileStream binStream = File.OpenWrite(@"LibTest\Binary_SaveTest.rbxl"))
bin.Save(binStream);
using (FileStream xmlStream = File.OpenWrite(@"LibTest\Xml_SaveTest.rbxlx"))
xml.Save(xmlStream);
bin.Save(@"Files\Binary_SaveTest.rbxl");
xml.Save(@"Files\Xml_SaveTest.rbxlx");
Console.WriteLine("Files saved! Pausing execution for debugger analysis...");
Debugger.Break();

View File

@ -0,0 +1,31 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>WinExe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<None Remove="Files\Binary.rbxl" />
<None Remove="Files\CoreScripts.rbxm" />
<None Remove="Files\Xml.rbxlx" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Files\Binary.rbxl">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="Files\Xml.rbxlx">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\RobloxFileFormat.csproj" />
</ItemGroup>
</Project>

View File

@ -115,14 +115,18 @@ internal static class Formatting
return Math.Abs(a - b) < epsilon;
}
public static byte[] ReadBuffer(this BinaryReader reader)
{
int len = reader.ReadInt32();
return reader.ReadBytes(len);
}
public static string ReadString(this BinaryReader reader, bool useIntLength)
{
if (!useIntLength)
return reader.ReadString();
int len = reader.ReadInt32();
byte[] buffer = reader.ReadBytes(len);
byte[] buffer = reader.ReadBuffer();
return Encoding.UTF8.GetString(buffer);
}
}

View File

@ -11,7 +11,7 @@ namespace RobloxFiles.XmlFormat
{
var errorHandler = new Func<string, Exception>((message) =>
{
string contents = $"XmlDataReader.{label}: {message}";
string contents = $"XmlRobloxFileReader.{label}: {message}";
return new Exception(contents);
});
@ -29,19 +29,25 @@ namespace RobloxFiles.XmlFormat
{
if (sharedString.Name == "SharedString")
{
XmlNode md5Node = sharedString.Attributes.GetNamedItem("md5");
XmlNode hashNode = sharedString.Attributes.GetNamedItem("md5");
if (md5Node == null)
if (hashNode == null)
throw error("Got a SharedString without an 'md5' attribute!");
string key = md5Node.InnerText;
string key = hashNode.InnerText;
string value = sharedString.InnerText.Replace("\n", "");
byte[] buffer = Convert.FromBase64String(value);
SharedString record = SharedString.FromBase64(value);
byte[] hash = Convert.FromBase64String(key);
var record = SharedString.FromBase64(value);
if (record.MD5_Key != key)
throw error("The provided md5 hash did not match with the md5 hash computed for the value!");
if (hash.Length != 16)
throw error($"SharedString base64 key '{key}' must decode to byte[16]!");
if (key != record.Key)
{
SharedString.Register(key, record.SharedValue);
record.Key = key;
}
file.SharedStrings.Add(key);
}

View File

@ -1,4 +1,5 @@
using System;
using System.Linq;
using System.Text;
using System.Xml;
@ -47,6 +48,9 @@ namespace RobloxFiles.XmlFormat
public static XmlNode WriteProperty(Property prop, XmlDocument doc, XmlRobloxFile file)
{
if (prop.Name == "Archivable")
return null;
string propType = prop.XmlToken;
if (propType == null)
@ -78,6 +82,7 @@ namespace RobloxFiles.XmlFormat
case PropertyType.String:
propType = (prop.HasRawBuffer ? "BinaryString" : "string");
break;
default: break;
}
}
@ -89,6 +94,19 @@ namespace RobloxFiles.XmlFormat
return null;
}
if (prop.Type == PropertyType.SharedString)
{
SharedString value = prop.CastValue<SharedString>();
if (value.ComputedKey == null)
{
var newShared = SharedString.FromBuffer(value.SharedValue);
value.Key = newShared.ComputedKey;
}
file.SharedStrings.Add(value.Key);
}
XmlElement propElement = doc.CreateElement(propType);
propElement.SetAttribute("name", prop.Name);
@ -102,12 +120,6 @@ namespace RobloxFiles.XmlFormat
propNode = newNode;
}
if (prop.Type == PropertyType.SharedString)
{
SharedString value = prop.CastValue<SharedString>();
file.SharedStrings.Add(value.MD5_Key);
}
return propNode;
}
@ -125,10 +137,18 @@ namespace RobloxFiles.XmlFormat
var props = instance.RefreshProperties();
foreach (string propName in props.Keys)
var orderedKeys = props
.OrderBy(pair => pair.Key)
.Select(pair => pair.Key);
foreach (string propName in orderedKeys)
{
Property prop = props[propName];
XmlNode propNode = WriteProperty(prop, doc, file);
if (propNode == null)
continue;
propsNode.AppendChild(propNode);
}
@ -151,12 +171,12 @@ namespace RobloxFiles.XmlFormat
var binaryWriter = XmlPropertyTokens.GetHandler<BinaryStringToken>();
var binaryBuffer = new Property("SharedString", PropertyType.String);
foreach (string md5 in file.SharedStrings)
foreach (string key in file.SharedStrings)
{
XmlElement sharedString = doc.CreateElement("SharedString");
sharedString.SetAttribute("md5", md5);
sharedString.SetAttribute("md5", key);
binaryBuffer.RawBuffer = SharedString.FindRecord(md5);
binaryBuffer.RawBuffer = SharedString.Find(key);
binaryWriter.WriteProperty(binaryBuffer, doc, sharedString);
sharedStrings.AppendChild(sharedString);

View File

@ -1,4 +1,5 @@
using System.Xml;
using System.Text;
using System.Xml;
using RobloxFiles.DataTypes;
namespace RobloxFiles.XmlFormat.PropertyTokens
@ -10,7 +11,7 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
public bool ReadProperty(Property prop, XmlNode token)
{
ProtectedString contents = token.InnerText;
prop.Type = PropertyType.String;
prop.Type = PropertyType.ProtectedString;
prop.Value = contents;
return true;
@ -18,16 +19,26 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
public void WriteProperty(Property prop, XmlDocument doc, XmlNode node)
{
string value = prop.CastValue<ProtectedString>();
ProtectedString value = prop.CastValue<ProtectedString>();
if (value.Contains("\r") || value.Contains("\n"))
if (value.IsCompiled)
{
XmlCDataSection cdata = doc.CreateCDataSection(value);
var binary = XmlPropertyTokens.GetHandler<BinaryStringToken>();
binary.WriteProperty(prop, doc, node);
}
else
{
string contents = Encoding.UTF8.GetString(value.RawBuffer);
if (contents.Contains("\r") || contents.Contains("\n"))
{
XmlCDataSection cdata = doc.CreateCDataSection(contents);
node.AppendChild(cdata);
}
else
{
node.InnerText = value;
node.InnerText = contents;
}
}
}
}

View File

@ -9,17 +9,25 @@ namespace RobloxFiles.XmlFormat.PropertyTokens
public bool ReadProperty(Property prop, XmlNode token)
{
string md5 = token.InnerText;
string key = token.InnerText;
prop.Type = PropertyType.SharedString;
prop.Value = new SharedString(md5);
prop.Value = new SharedString(key);
return true;
}
public void WriteProperty(Property prop, XmlDocument doc, XmlNode node)
{
SharedString value = prop.CastValue<SharedString>();
node.InnerText = value.MD5_Key;
var value = prop.CastValue<SharedString>();
string key = value.Key;
if (value.ComputedKey == null)
{
var newShared = SharedString.FromBuffer(value.SharedValue);
key = newShared.ComputedKey;
}
node.InnerText = key;
}
}
}

View File

@ -103,7 +103,7 @@ namespace RobloxFiles
foreach (Property sharedProp in sharedProps)
{
SharedString shared = sharedProp.CastValue<SharedString>();
SharedStrings.Add(shared.MD5_Key);
SharedStrings.Add(shared.Key);
}
}
else

View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Costura.Fody" version="3.3.3" targetFramework="net452" />
<package id="Fody" version="4.2.1" targetFramework="net452" developmentDependency="true" />
<package id="Costura.Fody" version="4.1.0" targetFramework="net472" />
<package id="Fody" version="6.2.0" targetFramework="net472" developmentDependency="true" />
<package id="Konscious.Security.Cryptography.Blake2" version="1.0.9" targetFramework="net472" />
<package id="lz4net" version="1.0.15.93" targetFramework="net452" />
<package id="System.Numerics.Vectors" version="4.5.0" targetFramework="net472" />
</packages>