Source2Roblox/Roblox-File-Format
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Large scale refactor to add class support!

Browse Source 
Instance classes are now strongly typed with real property fields that
are derived from the JSON API Dump! This required a lot of reworking
across the board:

- Classes and Enums are auto-generated in the 'Generated' folder now.
This is done using a custom built-in plugin, which can be found in
the Plugins folder of this project.
- Property objects are now tied to .NET's reflection system. Reading
and writing from them will try to redirect into a field of the
Instance they are bound to.
- Property types that were loosely defined now have proper data types
(such as Color3uint8, Content, ProtectedString, SharedString, etc)
- Fixed an error with the CFrame directional vectors.
- The binary PRNT chunk now writes instances in child->parent order.
- Enums are now generated correctly, with up-to-date values.
- INST chunks are now referred to as 'Classes' instead of 'Types'.
- Unary operator added to Vector2 and Vector3.
- CollectionService tags can now be manipulated per-instance using
the Instance.Tags member.
- The Instance.Archivable property now works correctly.
- XML files now save/load metadata correctly.
- Cleaned up the property tokens directory.

I probably missed a few things, but that's a general overview of
everything that changed.
This commit is contained in:
CloneTrooper1019
2019-06-30 17:01:19 -05:00
parent 8e01f33d6b
commit de8df15d3f
67 changed files with 6297 additions and 667 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
BinaryFormat/Chunks/INST.cs
View File

@ -1,12 +1,12 @@
using System.Collections.Generic;
using System.Linq;
using System;
using System.Collections.Generic;
namespace RobloxFiles.BinaryFormat.Chunks
{
public class INST : IBinaryFileChunk
{
public int TypeIndex { get; internal set; }
public string TypeName { get; internal set; }
public int ClassIndex { get; internal set; }
public string ClassName { get; internal set; }
public bool IsService { get; internal set; }
public List<bool> RootedServices { get; internal set; }
@ -14,17 +14,14 @@ namespace RobloxFiles.BinaryFormat.Chunks
public int NumInstances { get; internal set; }
public List<int> InstanceIds { get; internal set; }
public override string ToString()
{
return TypeName;
}
public override string ToString() => ClassName;
public void LoadFromReader(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
TypeIndex = reader.ReadInt32();
TypeName = reader.ReadString();
ClassIndex = reader.ReadInt32();
ClassName = reader.ReadString();
IsService = reader.ReadBoolean();
NumInstances = reader.ReadInt32();
@ -44,32 +41,30 @@ namespace RobloxFiles.BinaryFormat.Chunks
for (int i = 0; i < NumInstances; i++)
{
int instId = InstanceIds[i];
Type instType = Type.GetType($"RobloxFiles.{ClassName}") ?? typeof(Instance);
var inst = new Instance()
{
ClassName = TypeName,
IsService = IsService,
Referent = instId.ToString()
};
var inst = Activator.CreateInstance(instType) as Instance;
inst.Referent = instId.ToString();
inst.IsService = IsService;
if (IsService)
{
bool rooted = RootedServices[i];
inst.IsRootedService = rooted;
bool isRooted = RootedServices[i];
inst.Parent = (isRooted ? file : null);
}
file.Instances[instId] = inst;
}
file.Types[TypeIndex] = this;
file.Classes[ClassIndex] = this;
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
{
writer.StartWritingChunk(this);
writer.Write(TypeIndex);
writer.WriteString(TypeName);
writer.Write(ClassIndex);
writer.WriteString(ClassName);
writer.Write(IsService);
writer.Write(NumInstances);
@ -82,30 +77,13 @@ namespace RobloxFiles.BinaryFormat.Chunks
foreach (int instId in InstanceIds)
{
Instance service = file.Instances[instId];
writer.Write(service.IsRootedService);
bool isRooted = (service.Parent == file);
writer.Write(isRooted);
}
}
return writer.FinishWritingChunk();
}
internal static void ApplyTypeMap(BinaryRobloxFileWriter writer)
{
BinaryRobloxFile file = writer.File;
file.Instances = writer.Instances.ToArray();
var types = writer.TypeMap
.OrderBy(type => type.Key)
.Select(type => type.Value)
.ToArray();
for (int i = 0; i < types.Length; i++, file.NumTypes++)
{
INST type = types[i];
type.TypeIndex = i;
}
file.Types = types;
}
}
}

8
BinaryFormat/Chunks/PRNT.cs
View File

@ -27,21 +27,21 @@ namespace RobloxFiles.BinaryFormat.Chunks
Instance child = file.Instances[childId];
child.Parent = (parentId >= 0 ? file.Instances[parentId] : file);
child.ParentLocked = child.IsService;
}
}
public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer)
{
BinaryRobloxFile file = writer.File;
writer.StartWritingChunk(this);
Format = 0;
NumRelations = file.Instances.Length;
NumRelations = 0;
ChildrenIds = new List<int>();
ParentIds = new List<int>();
foreach (Instance inst in file.Instances)
foreach (Instance inst in writer.PostInstances)
{
Instance parent = inst.Parent;
@ -53,6 +53,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
ChildrenIds.Add(childId);
ParentIds.Add(parentId);
NumRelations++;
}
writer.Write(Format);

223
BinaryFormat/Chunks/PROP.cs
View File

@ -1,50 +1,62 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Reflection;
using System.Text;
using RobloxFiles.Enums;
using RobloxFiles.DataTypes;
using RobloxFiles.Utility;
using System.Text;
namespace RobloxFiles.BinaryFormat.Chunks
{
public class PROP : IBinaryFileChunk
{
public string Name { get; internal set; }
public int TypeIndex { get; internal set; }
public int ClassIndex { get; internal set; }
public string ClassName { get; private set; }
public PropertyType Type { get; internal set; }
public byte TypeId
{
get { return (byte)Type; }
internal set { Type = (PropertyType)value; }
}
public override string ToString()
{
return $"{Type} {ClassName}.{Name}";
}
public void LoadFromReader(BinaryRobloxFileReader reader)
{
BinaryRobloxFile file = reader.File;
TypeIndex = reader.ReadInt32();
ClassIndex = reader.ReadInt32();
Name = reader.ReadString();
TypeId = reader.ReadByte();
INST type = file.Types[TypeIndex];
Property[] props = new Property[type.NumInstances];
byte propType = reader.ReadByte();
Type = (PropertyType)propType;
INST inst = file.Classes[ClassIndex];
ClassName = inst.ClassName;
var ids = type.InstanceIds;
int instCount = type.NumInstances;
Property[] props = new Property[inst.NumInstances];
var ids = inst.InstanceIds;
int instCount = inst.NumInstances;
for (int i = 0; i < instCount; i++)
{
int id = ids[i];
Instance inst = file.Instances[id];
Instance instance = file.Instances[id];
Property prop = new Property(inst, this);
Property prop = new Property(instance, this);
props[i] = prop;
inst.AddProperty(ref prop);
instance.AddProperty(ref prop);
}
// Setup some short-hand functions for actions used during the read procedure.
@ -66,14 +78,36 @@ namespace RobloxFiles.BinaryFormat.Chunks
case PropertyType.String:
readProperties(i =>
{
string result = reader.ReadString();
string value = reader.ReadString();
// Leave an access point for the original byte sequence, in case this is a BinaryString.
// This will allow the developer to read the sequence without any mangling from C# strings.
byte[] buffer = reader.GetLastStringBuffer();
props[i].RawBuffer = buffer;
return result;
// Check if this is going to be casted as a BinaryString.
// BinaryStrings should use a type of byte[] instead.
Property prop = props[i];
Instance instance = prop.Instance;
Type instType = instance.GetType();
FieldInfo field = instType.GetField(Name);
if (field != null)
{
object result = value;
Type fieldType = field.FieldType;
if (fieldType == typeof(byte[]))
result = buffer;
return result;
}
else
{
return value;
}
});
break;
@ -213,15 +247,16 @@ namespace RobloxFiles.BinaryFormat.Chunks
case PropertyType.Quaternion:
// Temporarily load the rotation matrices into their properties.
// We'll update them to CFrames once we iterate over the position data.
float[][] matrices = new float[instCount][];
readProperties(i =>
for (int i = 0; i < instCount; i++)
{
byte b_OrientId = reader.ReadByte();
byte rawOrientId = reader.ReadByte();
if (b_OrientId > 0)
if (rawOrientId > 0)
{
// Make sure this value is in a safe range.
int orientId = (b_OrientId - 1) % 36;
int orientId = (rawOrientId - 1) % 36;
NormalId xColumn = (NormalId)(orientId / 6);
Vector3 R0 = Vector3.FromNormalId(xColumn);
@ -232,8 +267,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
// Compute R2 using the cross product of R0 and R1.
Vector3 R2 = R0.Cross(R1);
// Generate the rotation matrix and return it.
return new float[9]
// Generate the rotation matrix.
matrices[i] = new float[9]
{
R0.X, R0.Y, R0.Z,
R1.X, R1.Y, R1.Z,
@ -247,8 +282,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
Quaternion quaternion = new Quaternion(qx, qy, qz, qw);
var rotation = quaternion.ToCFrame();
return rotation.GetComponents();
matrices[i] = rotation.GetComponents();
}
else
{
@ -260,9 +294,9 @@ namespace RobloxFiles.BinaryFormat.Chunks
matrix[m] = value;
}
return matrix;
matrices[i] = matrix;
}
});
}
float[] CFrame_X = readFloats(),
CFrame_Y = readFloats(),
@ -270,25 +304,54 @@ namespace RobloxFiles.BinaryFormat.Chunks
readProperties(i =>
{
float[] matrix = props[i].Value as float[];
float[] matrix = matrices[i];
float x = CFrame_X[i],
y = CFrame_Y[i],
z = CFrame_Z[i];
float[] position = new float[3] { x, y, z };
float[] components = position.Concat(matrix).ToArray();
float[] components;
if (matrix.Length == 12)
{
matrix[0] = x;
matrix[1] = y;
matrix[2] = z;
components = matrix;
}
else
{
float[] position = new float[3] { x, y, z };
components = position.Concat(matrix).ToArray();
}
return new CFrame(components);
});
break;
case PropertyType.Enum:
// TODO: I want to map these values to actual Roblox enums, but I'll have to add an
// interpreter for the JSON API Dump to do it properly.
uint[] enums = reader.ReadUInts(instCount);
readProperties(i => enums[i]);
readProperties(i =>
{
Property prop = props[i];
Instance instance = prop.Instance;
Type instType = instance.GetType();
uint value = enums[i];
try
{
FieldInfo info = instType.GetField(Name, Property.BindingFlags);
return Enum.Parse(info.FieldType, value.ToInvariantString());
}
catch
{
Console.WriteLine($"Enum cast failed for {inst.ClassName}.{Name} using value {value}!");
return value;
}
});
break;
case PropertyType.Ref:
@ -345,9 +408,9 @@ namespace RobloxFiles.BinaryFormat.Chunks
B = reader.ReadFloat();
Color3 Value = new Color3(R, G, B);
byte[] Reserved = reader.ReadBytes(4);
int Envelope = reader.ReadInt32();
keypoints[key] = new ColorSequenceKeypoint(Time, Value, Reserved);
keypoints[key] = new ColorSequenceKeypoint(Time, Value, Envelope);
}
return new ColorSequence(keypoints);
@ -415,7 +478,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
g = Color3uint8_G[i],
b = Color3uint8_B[i];
return Color3.FromRGB(r, g, b);
Color3uint8 result = Color3.FromRGB(r, g, b);
return result;
});
break;
@ -434,6 +498,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
readProperties(i =>
{
uint key = SharedKeys[i];
return file.SharedStrings[key];
});
@ -455,11 +520,9 @@ namespace RobloxFiles.BinaryFormat.Chunks
foreach (int instId in inst.InstanceIds)
{
Instance instance = file.Instances[instId];
var props = instance.RefreshProperties();
var props = instance.Properties;
var propNames = props.Keys;
foreach (string propName in propNames)
foreach (string propName in props.Keys)
{
if (!propMap.ContainsKey(propName))
{
@ -469,7 +532,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
{
Name = prop.Name,
Type = prop.Type,
TypeIndex = inst.TypeIndex
ClassIndex = inst.ClassIndex
};
propMap.Add(propName, propChunk);
@ -484,13 +548,13 @@ namespace RobloxFiles.BinaryFormat.Chunks
{
BinaryRobloxFile file = writer.File;
INST inst = file.Types[TypeIndex];
INST inst = file.Classes[ClassIndex];
var props = new List<Property>();
foreach (int instId in inst.InstanceIds)
{
Instance instance = file.Instances[instId];
Property prop = instance.GetProperty(Name);
Property prop = instance.Properties[Name];
if (prop == null)
throw new Exception($"Property {Name} must be defined in {instance.GetFullName()}!");
@ -502,7 +566,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
}
writer.StartWritingChunk(this);
writer.Write(TypeIndex);
writer.Write(ClassIndex);
writer.WriteString(Name);
writer.Write(TypeId);
@ -526,7 +590,12 @@ namespace RobloxFiles.BinaryFormat.Chunks
break;
case PropertyType.Bool:
props.ForEach(prop => prop.WriteValue<bool>());
props.ForEach(prop =>
{
bool value = prop.CastValue<bool>();
writer.Write(value);
});
break;
case PropertyType.Int:
var ints = new List<int>();
@ -737,7 +806,15 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.ForEach(prop =>
{
uint value = prop.CastValue<uint>();
if (prop.Value is uint)
{
uint raw = prop.CastValue<uint>();
Enums.Add(raw);
return;
}
int signed = (int)prop.Value;
uint value = (uint)signed;
Enums.Add(value);
});
@ -805,7 +882,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
writer.Write(color.G);
writer.Write(color.B);
writer.Write(0);
writer.Write(keyPoint.Envelope);
}
});
@ -873,21 +950,20 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.ForEach(prop =>
{
Color3 value = prop.CastValue<Color3>();
byte r = (byte)(value.R * 255);
Color3uint8_R.Add(r);
byte g = (byte)(value.G * 255);
Color3uint8_G.Add(g);
byte b = (byte)(value.B * 255);
Color3uint8_B.Add(b);
Color3uint8 value = prop.CastValue<Color3uint8>();
Color3uint8_R.Add(value.R);
Color3uint8_G.Add(value.G);
Color3uint8_B.Add(value.B);
});
writer.Write(Color3uint8_R.ToArray());
writer.Write(Color3uint8_G.ToArray());
writer.Write(Color3uint8_B.ToArray());
byte[] rBuffer = Color3uint8_R.ToArray();
writer.Write(rBuffer);
byte[] gBuffer = Color3uint8_G.ToArray();
writer.Write(gBuffer);
byte[] bBuffer = Color3uint8_B.ToArray();
writer.Write(bBuffer);
break;
case PropertyType.Int64:
@ -918,27 +994,18 @@ namespace RobloxFiles.BinaryFormat.Chunks
props.ForEach(prop =>
{
uint sharedKey = 0;
string value = prop.CastValue<string>();
byte[] buffer = Encoding.UTF8.GetBytes(value);
using (MD5 md5 = MD5.Create())
SharedString shared = prop.CastValue<SharedString>();
string key = shared.MD5_Key;
if (!sstr.Lookup.ContainsKey(key))
{
byte[] hash = md5.ComputeHash(buffer);
string key = Convert.ToBase64String(hash);
if (!sstr.Lookup.ContainsKey(key))
{
uint id = (uint)(sstr.NumHashes++);
sstr.Strings.Add(id, value);
sstr.Lookup.Add(key, id);
}
sharedKey = sstr.Lookup[key];
uint id = (uint)(sstr.NumHashes++);
sstr.Strings.Add(id, shared);
sstr.Lookup.Add(key, id);
}
sharedKeys.Add(sharedKey);
uint hashId = sstr.Lookup[key];
sharedKeys.Add(hashId);
});
writer.WriteInterleaved(sharedKeys);

13
BinaryFormat/Chunks/SSTR.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using RobloxFiles.DataTypes;
namespace RobloxFiles.BinaryFormat.Chunks
{
@ -9,7 +10,7 @@ namespace RobloxFiles.BinaryFormat.Chunks
public int NumHashes;
public Dictionary<string, uint> Lookup = new Dictionary<string, uint>();
public Dictionary<uint, string> Strings = new Dictionary<uint, string>();
public Dictionary<uint, SharedString> Strings = new Dictionary<uint, SharedString>();
public void LoadFromReader(BinaryRobloxFileReader reader)
{
@ -25,10 +26,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
int length = reader.ReadInt32();
byte[] data = reader.ReadBytes(length);
string key = Convert.ToBase64String(md5);
string value = Convert.ToBase64String(data);
Lookup.Add(key, id);
SharedString value = SharedString.FromBuffer(data);
Lookup.Add(value.MD5_Key, id);
Strings.Add(id, value);
}
@ -49,8 +48,8 @@ namespace RobloxFiles.BinaryFormat.Chunks
byte[] md5 = Convert.FromBase64String(key);
writer.Write(md5);
string value = Strings[pair.Value];
byte[] buffer = Convert.FromBase64String(value);
SharedString value = Strings[pair.Value];
byte[] buffer = SharedString.FindRecord(value.MD5_Key);
writer.Write(buffer.Length);
writer.Write(buffer);