Source2Roblox/Roblox-File-Format
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Brought spec up to date, improvements to stability

Browse Source
This commit is contained in:
CloneTrooper1019
2020-07-12 20:19:30 -05:00
parent 7359b6efb7
commit 57fd3f8a25
29 changed files with 551 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
BinaryFormat/BinaryRobloxFile.cs
View File

@ -15,20 +15,22 @@ namespace RobloxFiles
// Header Specific
public const string MagicHeader = "<roblox!\x89\xff\x0d\x0a\x1a\x0a";
public ushort Version;
public uint NumClasses;
public uint NumInstances;
public long Reserved;
public ushort Version { get; internal set; }
public uint NumClasses { get; internal set; }
public uint NumInstances { get; internal set; }
public long Reserved { get; internal set; }
// Runtime Specific
public List<BinaryRobloxFileChunk> Chunks = new List<BinaryRobloxFileChunk>();
internal List<BinaryRobloxFileChunk> ChunksImpl = new List<BinaryRobloxFileChunk>();
public IReadOnlyList<BinaryRobloxFileChunk> Chunks => ChunksImpl;
public override string ToString() => GetType().Name;
public Instance[] Instances;
public INST[] Classes;
public Instance[] Instances { get; internal set; }
public INST[] Classes { get; internal set; }
internal META META = null;
internal SSTR SSTR = null;
internal SIGN SIGN = null;
public bool HasMetadata => (META != null);
public Dictionary<string, string> Metadata => META?.Data;
@ -36,6 +38,9 @@ namespace RobloxFiles
public bool HasSharedStrings => (SSTR != null);
public IReadOnlyDictionary<uint, SharedString> SharedStrings => SSTR?.Strings;
public bool HasSignatures => (SIGN != null);
public IReadOnlyList<Signature> Signatures => SIGN?.Signatures;
public BinaryRobloxFile()
{
Name = "BinaryRobloxFile";
@ -71,12 +76,10 @@ namespace RobloxFiles
{
try
{
BinaryRobloxFileChunk chunk = new BinaryRobloxFileChunk(reader);
string chunkType = chunk.ChunkType;
var chunk = new BinaryRobloxFileChunk(reader);
IBinaryFileChunk handler = null;
switch (chunkType)
switch (chunk.ChunkType)
{
case "INST":
handler = new INST();
@ -93,22 +96,27 @@ namespace RobloxFiles
case "SSTR":
handler = new SSTR();
break;
case "SIGN":
handler = new SIGN();
break;
case "END\0":
Chunks.Add(chunk);
ChunksImpl.Add(chunk);
reading = false;
break;
default:
Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", chunkType);
case string unhandled:
Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", unhandled);
break;
default: break;
}
if (handler != null)
{
using (BinaryRobloxFileReader dataReader = chunk.GetDataReader(this))
handler.LoadFromReader(dataReader);
chunk.Handler = handler;
Chunks.Add(chunk);
using (var dataReader = chunk.GetDataReader(this))
handler.Load(dataReader);
ChunksImpl.Add(chunk);
}
}
catch (EndOfStreamException)
@ -129,7 +137,7 @@ namespace RobloxFiles
{
// Clear the existing data.
Referent = "-1";
Chunks.Clear();
ChunksImpl.Clear();
NumInstances = 0;
NumClasses = 0;
@ -148,7 +156,7 @@ namespace RobloxFiles
// Write the PROP chunks.
foreach (INST inst in Classes)
{
Dictionary<string, PROP> props = PROP.CollectProperties(writer, inst);
var props = PROP.CollectProperties(writer, inst);
foreach (string propName in props.Keys)
{
@ -158,26 +166,23 @@ namespace RobloxFiles
}
// Write the PRNT chunk.
PRNT parents = new PRNT();
var parents = new PRNT();
writer.SaveChunk(parents);
// Write the SSTR chunk.
if (HasSharedStrings)
{
var sharedStrings = SSTR.SaveAsChunk(writer);
Chunks.Insert(0, sharedStrings);
}
writer.SaveChunk(SSTR, 0);
// Write the META chunk.
if (HasMetadata)
{
var metaChunk = META.SaveAsChunk(writer);
Chunks.Insert(0, metaChunk);
}
writer.SaveChunk(META, 0);
// Write the END_ chunk.
var endChunk = writer.WriteEndChunk();
Chunks.Add(endChunk);
// Write the SIGN chunk.
if (HasSignatures)
writer.SaveChunk(SIGN);
// Write the END chunk.
writer.WriteChunk("END", "</roblox>");
}
//////////////////////////////////////////////////////////////////////////

10
BinaryFormat/Chunks/INST.cs
View File

@ -16,7 +16,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
public override string ToString() => ClassName;
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
@ -59,10 +59,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
file.Classes[ClassIndex] = this;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
writer.Write(ClassIndex);
writer.WriteString(ClassName);
@ -72,7 +70,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
if (IsService)
{
BinaryRobloxFile file = writer.File;
var file = writer.File;
foreach (int instId in InstanceIds)
{
@ -80,8 +78,6 @@ namespace RobloxFiles.BinaryFormat.Chunks
writer.Write(service.Parent == file);
}
}
return writer.FinishWritingChunk();
}
}
}

13
BinaryFormat/Chunks/META.cs
View File

@ -6,7 +6,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
{
public Dictionary<string, string> Data = new Dictionary<string, string>();
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
int numEntries = reader.ReadInt32();
@ -21,18 +21,15 @@ namespace RobloxFiles.BinaryFormat.Chunks
file.META = this;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
writer.Write(Data.Count);
foreach (var kvPair in Data)
foreach (var pair in Data)
{
writer.WriteString(kvPair.Key);
writer.WriteString(kvPair.Value);
writer.WriteString(pair.Key);
writer.WriteString(pair.Value);
}
return writer.FinishWritingChunk();
}
}
}

56
BinaryFormat/Chunks/PRNT.cs
View File

@ -1,29 +1,29 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
namespace RobloxFiles.BinaryFormat.Chunks
{
public class PRNT : IBinaryFileChunk
{
public byte Format { get; private set; }
public int NumRelations { get; private set; }
private const byte FORMAT = 0;
public List<int> ChildrenIds { get; private set; }
public List<int> ParentIds { get; private set; }
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
Format = reader.ReadByte();
NumRelations = reader.ReadInt32();
byte format = reader.ReadByte();
int idCount = reader.ReadInt32();
ChildrenIds = reader.ReadInstanceIds(NumRelations);
ParentIds = reader.ReadInstanceIds(NumRelations);
if (format != FORMAT)
throw new Exception($"Unexpected PRNT format: {format} (expected {FORMAT}!)");
var childIds = reader.ReadInstanceIds(idCount);
var parentIds = reader.ReadInstanceIds(idCount);
for (int i = 0; i < NumRelations; i++)
for (int i = 0; i < idCount; i++)
{
int childId = ChildrenIds[i];
int parentId = ParentIds[i];
int childId = childIds[i];
int parentId = parentIds[i];
Instance child = file.Instances[childId];
child.Parent = (parentId >= 0 ? file.Instances[parentId] : file);
@ -31,15 +31,13 @@ namespace RobloxFiles.BinaryFormat.Chunks
}
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
var postInstances = writer.PostInstances;
var idCount = postInstances.Count;
Format = 0;
NumRelations = 0;
ChildrenIds = new List<int>();
ParentIds = new List<int>();
var childIds = new List<int>();
var parentIds = new List<int>();
foreach (Instance inst in writer.PostInstances)
{
@ -51,19 +49,15 @@ namespace RobloxFiles.BinaryFormat.Chunks
if (parent != null)
parentId = int.Parse(parent.Referent);
ChildrenIds.Add(childId);
ParentIds.Add(parentId);
NumRelations++;
childIds.Add(childId);
parentIds.Add(parentId);
}
writer.Write(Format);
writer.Write(NumRelations);
writer.Write(FORMAT);
writer.Write(idCount);
writer.WriteInstanceIds(ChildrenIds);
writer.WriteInstanceIds(ParentIds);
return writer.FinishWritingChunk();
writer.WriteInstanceIds(childIds);
writer.WriteInstanceIds(parentIds);
}
}
}

41
BinaryFormat/Chunks/PROP.cs
View File

@ -29,7 +29,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
return $"{Type} {ClassName}.{Name}";
}
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
@ -71,7 +71,6 @@ namespace RobloxFiles.BinaryFormat.Chunks
}
});
// Read the property data based on the property type.
switch (Type)
{
case PropertyType.String:
@ -500,13 +499,22 @@ namespace RobloxFiles.BinaryFormat.Chunks
readProperties(i =>
{
uint key = SharedKeys[i];
return file.SharedStrings[key];
});
break;
case PropertyType.ProtectedString:
readProperties(i =>
{
int length = reader.ReadInt32();
byte[] buffer = reader.ReadBytes(length);
return new ProtectedString(buffer);
});
break;
default:
Console.WriteLine("Unhandled property type: {0}!", Type);
Console.Error.WriteLine("Unhandled property type: {0}!", Type);
break;
//
}
@ -546,7 +554,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
return propMap;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
BinaryRobloxFile file = writer.File;
@ -567,9 +575,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.Add(prop);
}
writer.StartWritingChunk(this);
writer.Write(ClassIndex);
writer.WriteString(Name);
writer.Write(TypeId);
@ -996,12 +1002,12 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.ForEach(prop =>
{
SharedString shared = prop.CastValue<SharedString>();
string key = shared.MD5_Key;
var shared = prop.CastValue<SharedString>();
string key = shared.Key;
if (!sstr.Lookup.ContainsKey(key))
{
uint id = (uint)(sstr.NumHashes++);
uint id = (uint)(sstr.Lookup.Count);
sstr.Strings.Add(id, shared);
sstr.Lookup.Add(key, id);
}
@ -1012,10 +1018,19 @@ namespace RobloxFiles.BinaryFormat.Chunks
writer.WriteInterleaved(sharedKeys);
break;
//
case PropertyType.ProtectedString:
props.ForEach(prop =>
{
var protect = prop.CastValue<ProtectedString>();
byte[] buffer = protect.RawBuffer;
writer.Write(buffer.Length);
writer.Write(buffer);
});
break;
default: break;
}
return writer.FinishWritingChunk();
}
}
}

56
BinaryFormat/Chunks/SIGN.cs Normal file
View File

@ -0,0 +1,56 @@
namespace RobloxFiles.BinaryFormat.Chunks
{
public struct Signature
{
public int Version;
public long Id;
public int Length;
public byte[] Data;
}
public class SIGN : IBinaryFileChunk
{
public Signature[] Signatures;
public void Load(BinaryRobloxFileReader reader)
{
int numSignatures = reader.ReadInt32();
Signatures = new Signature[numSignatures];
for (int i = 0; i < numSignatures; i++)
{
var signature = new Signature
{
Version = reader.ReadInt32(),
Id = reader.ReadInt64(),
Length = reader.ReadInt32(),
};
signature.Data = reader.ReadBytes(signature.Length);
Signatures[i] = signature;
}
var file = reader.File;
file.SIGN = this;
}
public void Save(BinaryRobloxFileWriter writer)
{
writer.Write(Signatures.Length);
for (int i = 0; i < Signatures.Length; i++)
{
var signature = Signatures[i];
signature.Length = signature.Data.Length;
writer.Write(signature.Version);
writer.Write(signature.Id);
writer.Write(signature.Length);
writer.Write(signature.Data);
}
}
}
}

44
BinaryFormat/Chunks/SSTR.cs
View File

@ -6,56 +6,54 @@ namespace RobloxFiles.BinaryFormat.Chunks
{
public class SSTR : IBinaryFileChunk
{
public int Version;
public int NumHashes;
private const int FORMAT = 0;
public Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
public Dictionary<uint, SharedString> Strings = new Dictionary<uint, SharedString>();
internal Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
internal Dictionary<uint, SharedString> Strings = new Dictionary<uint, SharedString>();
public void LoadFromReader(BinaryRobloxFileReader reader)
public void Load(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
Version = reader.ReadInt32();
NumHashes = reader.ReadInt32();
int format = reader.ReadInt32();
int numHashes = reader.ReadInt32();
for (uint id = 0; id < NumHashes; id++)
if (format != FORMAT)
throw new Exception($"Unexpected SSTR format: {format} (expected {FORMAT}!)");
for (uint id = 0; id < numHashes; id++)
{
byte[] md5 = reader.ReadBytes(16);
int length = reader.ReadInt32();
byte[] data = reader.ReadBytes(length);
byte[] hash = reader.ReadBytes(16);
string key = Convert.ToBase64String(hash);
byte[] data = reader.ReadBuffer();
SharedString value = SharedString.FromBuffer(data);
Lookup.Add(value.MD5_Key, id);
Lookup.Add(key, id);
Strings.Add(id, value);
}
file.SSTR = this;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
public void Save(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
writer.Write(Version);
writer.Write(NumHashes);
writer.Write(FORMAT);
writer.Write(Lookup.Count);
foreach (var pair in Lookup)
{
string key = pair.Key;
byte[] md5 = Convert.FromBase64String(key);
writer.Write(md5);
byte[] hash = Convert.FromBase64String(key);
writer.Write(hash);
SharedString value = Strings[pair.Value];
byte[] buffer = SharedString.FindRecord(value.MD5_Key);
byte[] buffer = SharedString.Find(value.Key);
writer.Write(buffer.Length);
writer.Write(buffer);
}
return writer.FinishWritingChunk();
}
}
}

61
BinaryFormat/IO/BinaryFileWriter.cs
View File

@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
@ -18,14 +17,16 @@ namespace RobloxFiles.BinaryFormat
public string ChunkType { get; private set; }
public long ChunkStart { get; private set; }
public Dictionary<string, INST> ClassMap;
public readonly BinaryRobloxFile File;
// Dictionary mapping ClassNames to their INST chunks.
private readonly Dictionary<string, INST> ClassMap;
// Instances in parent->child order
public List<Instance> Instances;
private readonly List<Instance> Instances;
// Instances in child->parent order
public List<Instance> PostInstances;
internal List<Instance> PostInstances { get; private set; }
public BinaryRobloxFileWriter(BinaryRobloxFile file, Stream workBuffer = null) : base(workBuffer ?? new MemoryStream())
{
@ -104,13 +105,13 @@ namespace RobloxFiles.BinaryFormat
}
// Encodes an int for an interleaved buffer.
private int EncodeInt(int value)
private static int EncodeInt(int value)
{
return (value << 1) ^ (value >> 31);
}
// Encodes a float for an interleaved buffer.
private float EncodeFloat(float value)
private static float EncodeFloat(float value)
{
byte[] buffer = BitConverter.GetBytes(value);
uint bits = BitConverter.ToUInt32(buffer, 0);
@ -171,9 +172,9 @@ namespace RobloxFiles.BinaryFormat
Instances.Add(instance);
string className = instance.ClassName;
INST inst = null;
INST inst;
if (!ClassMap.ContainsKey(className))
if (!ClassMap.TryGetValue(className, out inst))
{
inst = new INST()
{
@ -184,11 +185,7 @@ namespace RobloxFiles.BinaryFormat
ClassMap.Add(className, inst);
}
else
{
inst = ClassMap[className];
}
inst.NumInstances++;
inst.InstanceIds.Add(instId);
@ -222,7 +219,7 @@ namespace RobloxFiles.BinaryFormat
}
// Marks that we are writing a chunk.
public bool StartWritingChunk(string chunkType)
private bool StartWritingChunk(string chunkType)
{
if (chunkType.Length != 4)
throw new Exception("BinaryFileWriter.StartWritingChunk - ChunkType length should be 4!");
@ -241,7 +238,7 @@ namespace RobloxFiles.BinaryFormat
}
// Marks that we are writing a chunk.
public bool StartWritingChunk(IBinaryFileChunk chunk)
private bool StartWritingChunk(IBinaryFileChunk chunk)
{
if (!WritingChunk)
{
@ -257,7 +254,7 @@ namespace RobloxFiles.BinaryFormat
}
// Compresses the data that was written into a BinaryRobloxFileChunk and writes it.
public BinaryRobloxFileChunk FinishWritingChunk(bool compress = true)
private BinaryRobloxFileChunk FinishWritingChunk(bool compress = true)
{
if (!WritingChunk)
throw new Exception($"BinaryRobloxFileWriter: Cannot finish writing a chunk without starting one!");
@ -283,18 +280,36 @@ namespace RobloxFiles.BinaryFormat
return chunk;
}
public void SaveChunk(IBinaryFileChunk handler)
internal BinaryRobloxFileChunk SaveChunk(IBinaryFileChunk handler, int insertPos = -1)
{
var chunk = handler.SaveAsChunk(this);
File.Chunks.Add(chunk);
StartWritingChunk(handler);
handler.Save(this);
var chunk = FinishWritingChunk();
if (insertPos >= 0)
File.ChunksImpl.Insert(insertPos, chunk);
else
File.ChunksImpl.Add(chunk);
return chunk;
}
public BinaryRobloxFileChunk WriteEndChunk()
internal BinaryRobloxFileChunk WriteChunk(string chunkType, string content, bool compress = false)
{
StartWritingChunk("END\0");
WriteString("</roblox>", true);
if (chunkType.Length > 4)
chunkType = chunkType.Substring(0, 4);
return FinishWritingChunk(false);
while (chunkType.Length < 4)
chunkType += '\0';
StartWritingChunk(chunkType);
WriteString(content);
var chunk = FinishWritingChunk(compress);
File.ChunksImpl.Add(chunk);
return chunk;
}
}
}