From 47112242e7f07edf03532b592e87b4b472972e9b Mon Sep 17 00:00:00 2001 From: CloneTrooper1019 Date: Fri, 7 Jun 2019 22:43:28 -0500 Subject: [PATCH] Added write support for binary files! Holy cow, this took a lot of work. I think I may need to do a few more things before I consider this a 1.0 release, but I'm glad to have finally overcome this hurdle! --- BinaryFormat/BinaryFileChunk.cs | 84 ++- BinaryFormat/BinaryRobloxFile.cs | 164 +++- BinaryFormat/Chunks/INST.cs | 111 ++- BinaryFormat/Chunks/META.cs | 37 +- BinaryFormat/Chunks/PRNT.cs | 66 +- BinaryFormat/Chunks/PROP.cs | 700 +++++++++++++++--- BinaryFormat/Chunks/SSTR.cs | 55 +- BinaryFormat/{ => IO}/BinaryFileReader.cs | 13 +- BinaryFormat/IO/BinaryFileWriter.cs | 240 ++++++ DataTypes/CFrame.cs | 50 ++ DataTypes/Vector3.cs | 23 +- Interfaces/IBinaryFileChunk.cs | 8 + .../IXmlPropertyToken.cs | 0 RobloxFile.cs | 2 +- RobloxFileFormat.csproj | 7 +- Tree/Instance.cs | 14 +- Tree/Property.cs | 30 +- XmlFormat/IO/XmlFileReader.cs | 2 +- XmlFormat/IO/XmlFileWriter.cs | 8 +- XmlFormat/PropertyTokens/Tokens/Ref.cs | 2 +- XmlFormat/PropertyTokens/XmlPropertyTokens.cs | 7 +- XmlFormat/XmlRobloxFile.cs | 4 +- 22 files changed, 1405 insertions(+), 222 deletions(-) rename BinaryFormat/{ => IO}/BinaryFileReader.cs (90%) create mode 100644 BinaryFormat/IO/BinaryFileWriter.cs create mode 100644 Interfaces/IBinaryFileChunk.cs rename {XmlFormat/PropertyTokens => Interfaces}/IXmlPropertyToken.cs (100%) diff --git a/BinaryFormat/BinaryFileChunk.cs b/BinaryFormat/BinaryFileChunk.cs index 4a8376d..400cb12 100644 --- a/BinaryFormat/BinaryFileChunk.cs +++ b/BinaryFormat/BinaryFileChunk.cs @@ -12,7 +12,7 @@ namespace RobloxFiles.BinaryFormat public class BinaryRobloxFileChunk { public readonly string ChunkType; - public readonly byte[] Reserved; + public readonly int Reserved; public readonly int CompressedSize; public readonly int Size; @@ -21,27 +21,33 @@ namespace RobloxFiles.BinaryFormat public readonly byte[] Data; public bool HasCompressedData => (CompressedSize > 0); + public IBinaryFileChunk Handler { get; internal set; } - public BinaryRobloxFileReader GetDataReader() + public bool HasWriteBuffer { get; private set; } + public byte[] WriteBuffer { get; private set; } + + public BinaryRobloxFileReader GetDataReader(BinaryRobloxFile file) { MemoryStream buffer = new MemoryStream(Data); - return new BinaryRobloxFileReader(buffer); + return new BinaryRobloxFileReader(file, buffer); } public override string ToString() { - return ChunkType + " Chunk [" + Size + " bytes]"; + string chunkType = ChunkType.Replace('\0', ' '); + int bytes = (HasCompressedData ? CompressedSize : Size); + + return $"'{chunkType}' Chunk ({bytes} bytes)"; } public BinaryRobloxFileChunk(BinaryRobloxFileReader reader) { - byte[] bChunkType = reader.ReadBytes(4); - ChunkType = Encoding.ASCII.GetString(bChunkType); + byte[] rawChunkType = reader.ReadBytes(4); + ChunkType = Encoding.ASCII.GetString(rawChunkType); CompressedSize = reader.ReadInt32(); Size = reader.ReadInt32(); - - Reserved = reader.ReadBytes(4); + Reserved = reader.ReadInt32(); if (HasCompressedData) { @@ -53,5 +59,67 @@ namespace RobloxFiles.BinaryFormat Data = reader.ReadBytes(Size); } } + + public BinaryRobloxFileChunk(BinaryRobloxFileWriter writer, bool compress = true) + { + if (!writer.WritingChunk) + throw new Exception("BinaryRobloxFileChunk: Supplied writer must have WritingChunk set to true."); + + Stream stream = writer.BaseStream; + + using (BinaryReader reader = new BinaryReader(stream, Encoding.UTF8, true)) + { + long length = (stream.Position - writer.ChunkStart); + stream.Position = writer.ChunkStart; + + Size = (int)length; + Data = reader.ReadBytes(Size); + } + + CompressedData = LZ4Codec.Encode(Data, 0, Size); + CompressedSize = CompressedData.Length; + + if (!compress || CompressedSize > Size) + { + CompressedSize = 0; + CompressedData = new byte[0]; + } + + ChunkType = writer.ChunkType; + Reserved = 0; + } + + public void WriteChunk(BinaryRobloxFileWriter writer) + { + // Record where we are when we start writing. + var stream = writer.BaseStream; + long startPos = stream.Position; + + // Write the chunk's data. + writer.WriteString(ChunkType, true); + + writer.Write(CompressedSize); + writer.Write(Size); + + writer.Write(Reserved); + + if (CompressedSize > 0) + writer.Write(CompressedData); + else + writer.Write(Data); + + // Capture the data we wrote into a byte[] array. + long endPos = stream.Position; + int length = (int)(endPos - startPos); + + using (MemoryStream buffer = new MemoryStream()) + { + stream.Position = startPos; + stream.CopyTo(buffer, length); + + WriteBuffer = buffer.ToArray(); + HasWriteBuffer = true; + } + } } } diff --git a/BinaryFormat/BinaryRobloxFile.cs b/BinaryFormat/BinaryRobloxFile.cs index e1bbbb2..5c07b87 100644 --- a/BinaryFormat/BinaryRobloxFile.cs +++ b/BinaryFormat/BinaryRobloxFile.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using RobloxFiles.BinaryFormat.Chunks; @@ -15,7 +16,7 @@ namespace RobloxFiles.BinaryFormat public ushort Version; public uint NumTypes; public uint NumInstances; - public byte[] Reserved; + public long Reserved; // Runtime Specific public List Chunks = new List(); @@ -24,19 +25,26 @@ namespace RobloxFiles.BinaryFormat public Instance[] Instances; public INST[] Types; - public Dictionary Metadata; - public Dictionary SharedStrings; + internal META META = null; + internal SSTR SSTR = null; + + public bool HasMetadata => (META != null); + public Dictionary Metadata => META?.Data; + + public bool HasSharedStrings => (SSTR != null); + public Dictionary SharedStrings => SSTR?.Strings; internal BinaryRobloxFile() { Name = "BinaryRobloxFile"; ParentLocked = true; + Referent = "-1"; } protected override void ReadFile(byte[] contents) { using (MemoryStream file = new MemoryStream(contents)) - using (BinaryRobloxFileReader reader = new BinaryRobloxFileReader(file)) + using (BinaryRobloxFileReader reader = new BinaryRobloxFileReader(this, file)) { // Verify the signature of the file. byte[] binSignature = reader.ReadBytes(14); @@ -49,51 +57,56 @@ namespace RobloxFiles.BinaryFormat Version = reader.ReadUInt16(); NumTypes = reader.ReadUInt32(); NumInstances = reader.ReadUInt32(); - Reserved = reader.ReadBytes(8); + Reserved = reader.ReadInt64(); // Begin reading the file chunks. bool reading = true; Types = new INST[NumTypes]; Instances = new Instance[NumInstances]; - + while (reading) { try { BinaryRobloxFileChunk chunk = new BinaryRobloxFileChunk(reader); - Chunks.Add(chunk); + string chunkType = chunk.ChunkType; - switch (chunk.ChunkType) + IBinaryFileChunk handler = null; + + switch (chunkType) { case "INST": - INST type = new INST(chunk); - type.Allocate(this); + handler = new INST(); break; case "PROP": - PROP prop = new PROP(chunk); - prop.ReadProperties(this); + handler = new PROP(); break; case "PRNT": - PRNT hierarchy = new PRNT(chunk); - hierarchy.Assemble(this); + handler = new PRNT(); break; case "META": - META meta = new META(chunk); - Metadata = meta.Data; + handler = new META(); break; case "SSTR": - SSTR shared = new SSTR(chunk); - SharedStrings = shared.Strings; + handler = new SSTR(); break; case "END\0": reading = false; break; default: - Console.WriteLine("BinaryRobloxFile: Unhandled chunk type: {0}!", chunk.ChunkType); - Chunks.Remove(chunk); + Console.WriteLine("BinaryRobloxFile - Unhandled chunk-type: {0}!", chunkType); break; } + + if (handler != null) + { + using (BinaryRobloxFileReader dataReader = chunk.GetDataReader(this)) + handler.LoadFromReader(dataReader); + + chunk.Handler = handler; + Chunks.Add(chunk); + } } catch (EndOfStreamException) { @@ -105,7 +118,116 @@ namespace RobloxFiles.BinaryFormat public override void Save(Stream stream) { - throw new NotImplementedException("Not implemented yet!"); + ////////////////////////////////////////////////////////////////////////// + // Generate the chunk data. + ////////////////////////////////////////////////////////////////////////// + + using (var writer = new BinaryRobloxFileWriter(this)) + { + // Clear the existing data. + Referent = "-1"; + Chunks.Clear(); + + NumInstances = 0; + NumTypes = 0; + + if (HasSharedStrings) + { + SSTR.NumHashes = 0; + SSTR.Lookup.Clear(); + SSTR.Strings.Clear(); + } + + // Write the META chunk. + if (HasMetadata) + { + var metaChunk = META.SaveAsChunk(writer); + Chunks.Add(metaChunk); + } + + // Record all instances and types. + writer.RecordInstances(Children); + + // Apply the type values. + INST.ApplyTypeMap(writer); + + // Write the INST chunks. + foreach (INST type in Types) + { + var instChunk = type.SaveAsChunk(writer); + Chunks.Add(instChunk); + } + + // Write the PROP chunks. + foreach (INST type in Types) + { + Dictionary props = PROP.CollectProperties(writer, type); + + foreach (string propName in props.Keys) + { + PROP prop = props[propName]; + + var chunk = prop.SaveAsChunk(writer); + Chunks.Add(chunk); + } + } + + // Write the PRNT chunk. + PRNT parents = new PRNT(); + + var parentChunk = parents.SaveAsChunk(writer); + Chunks.Add(parentChunk); + + // Write the SSTR chunk. + if (HasSharedStrings) + { + var sharedStrings = SSTR.SaveAsChunk(writer); + Chunks.Insert(0, sharedStrings); + } + + // Write the END_ chunk. + writer.StartWritingChunk("END\0"); + writer.WriteString("", true); + + var endChunk = writer.FinishWritingChunk(false); + Chunks.Add(endChunk); + } + + ////////////////////////////////////////////////////////////////////////// + // Write the chunks with the header & footer data + ////////////////////////////////////////////////////////////////////////// + + using (BinaryWriter writer = new BinaryWriter(stream)) + { + stream.Position = 0; + stream.SetLength(0); + + byte[] magicHeader = MagicHeader + .Select(ch => (byte)ch) + .ToArray(); + + writer.Write(magicHeader); + + writer.Write(Version); + writer.Write(NumTypes); + writer.Write(NumInstances); + + // Write the 8 reserved-bytes. + writer.Write(0L); + + // Write all of the chunks. + foreach (BinaryRobloxFileChunk chunk in Chunks) + { + byte[] chunkType = Encoding.ASCII.GetBytes(chunk.ChunkType); + + if (chunk.HasWriteBuffer) + { + byte[] writeBuffer = chunk.WriteBuffer; + writer.Write(writeBuffer); + } + + } + } } } } \ No newline at end of file diff --git a/BinaryFormat/Chunks/INST.cs b/BinaryFormat/Chunks/INST.cs index 2c97e54..fff8620 100644 --- a/BinaryFormat/Chunks/INST.cs +++ b/BinaryFormat/Chunks/INST.cs @@ -1,40 +1,111 @@ -namespace RobloxFiles.BinaryFormat.Chunks +using System.Collections.Generic; +using System.Linq; + +namespace RobloxFiles.BinaryFormat.Chunks { - public class INST + public class INST : IBinaryFileChunk { - public readonly int TypeIndex; - public readonly string TypeName; - public readonly bool IsService; - public readonly int NumInstances; - public readonly int[] InstanceIds; + public int TypeIndex { get; internal set; } + public string TypeName { get; internal set; } + + public bool IsService { get; internal set; } + public List RootedServices { get; internal set; } + + public int NumInstances { get; internal set; } + public List InstanceIds { get; internal set; } public override string ToString() { return TypeName; } - public INST(BinaryRobloxFileChunk chunk) + public void LoadFromReader(BinaryRobloxFileReader reader) { - using (BinaryRobloxFileReader reader = chunk.GetDataReader()) - { - TypeIndex = reader.ReadInt32(); - TypeName = reader.ReadString(); - IsService = reader.ReadBoolean(); + BinaryRobloxFile file = reader.File; - NumInstances = reader.ReadInt32(); - InstanceIds = reader.ReadInstanceIds(NumInstances); + TypeIndex = reader.ReadInt32(); + TypeName = reader.ReadString(); + IsService = reader.ReadBoolean(); + + NumInstances = reader.ReadInt32(); + InstanceIds = reader.ReadInstanceIds(NumInstances); + + if (IsService) + { + RootedServices = new List(); + + for (int i = 0; i < NumInstances; i++) + { + bool isRooted = reader.ReadBoolean(); + RootedServices.Add(isRooted); + } } - } - public void Allocate(BinaryRobloxFile file) - { - foreach (int instId in InstanceIds) + for (int i = 0; i < NumInstances; i++) { - Instance inst = new Instance() { ClassName = TypeName }; + int instId = InstanceIds[i]; + + var inst = new Instance() + { + ClassName = TypeName, + IsService = IsService, + Referent = instId.ToString() + }; + + if (IsService) + { + bool rooted = RootedServices[i]; + inst.IsRootedService = rooted; + } + file.Instances[instId] = inst; } file.Types[TypeIndex] = this; } + + public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + { + writer.StartWritingChunk(this); + + writer.Write(TypeIndex); + writer.WriteString(TypeName); + + writer.Write(IsService); + writer.Write(NumInstances); + writer.WriteInstanceIds(InstanceIds); + + if (IsService) + { + BinaryRobloxFile file = writer.File; + + foreach (int instId in InstanceIds) + { + Instance service = file.Instances[instId]; + writer.Write(service.IsRootedService); + } + } + + return writer.FinishWritingChunk(); + } + + internal static void ApplyTypeMap(BinaryRobloxFileWriter writer) + { + BinaryRobloxFile file = writer.File; + file.Instances = writer.Instances.ToArray(); + + var types = writer.TypeMap + .OrderBy(type => type.Key) + .Select(type => type.Value) + .ToArray(); + + for (int i = 0; i < types.Length; i++, file.NumTypes++) + { + INST type = types[i]; + type.TypeIndex = i; + } + + file.Types = types; + } } } diff --git a/BinaryFormat/Chunks/META.cs b/BinaryFormat/Chunks/META.cs index a4d1bb2..438752f 100644 --- a/BinaryFormat/Chunks/META.cs +++ b/BinaryFormat/Chunks/META.cs @@ -2,24 +2,37 @@ namespace RobloxFiles.BinaryFormat.Chunks { - public class META + public class META : IBinaryFileChunk { - public int NumEntries; public Dictionary Data = new Dictionary(); - public META(BinaryRobloxFileChunk chunk) + public void LoadFromReader(BinaryRobloxFileReader reader) { - using (BinaryRobloxFileReader reader = chunk.GetDataReader()) - { - NumEntries = reader.ReadInt32(); + BinaryRobloxFile file = reader.File; + int numEntries = reader.ReadInt32(); - for (int i = 0; i < NumEntries; i++) - { - string key = reader.ReadString(); - string value = reader.ReadString(); - Data.Add(key, value); - } + for (int i = 0; i < numEntries; i++) + { + string key = reader.ReadString(); + string value = reader.ReadString(); + Data.Add(key, value); } + + file.META = this; + } + + public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + { + writer.StartWritingChunk(this); + writer.Write(Data.Count); + + foreach (var kvPair in Data) + { + writer.WriteString(kvPair.Key); + writer.WriteString(kvPair.Value); + } + + return writer.FinishWritingChunk(); } } } diff --git a/BinaryFormat/Chunks/PRNT.cs b/BinaryFormat/Chunks/PRNT.cs index 5c96cb0..4b6fd4a 100644 --- a/BinaryFormat/Chunks/PRNT.cs +++ b/BinaryFormat/Chunks/PRNT.cs @@ -1,27 +1,25 @@ -namespace RobloxFiles.BinaryFormat.Chunks +using System.Collections.Generic; + +namespace RobloxFiles.BinaryFormat.Chunks { - public class PRNT + public class PRNT : IBinaryFileChunk { - public readonly byte Format; - public readonly int NumRelations; + public byte Format { get; private set; } + public int NumRelations { get; private set; } - public readonly int[] ChildrenIds; - public readonly int[] ParentIds; + public List ChildrenIds { get; private set; } + public List ParentIds { get; private set; } - public PRNT(BinaryRobloxFileChunk chunk) + public void LoadFromReader(BinaryRobloxFileReader reader) { - using (BinaryRobloxFileReader reader = chunk.GetDataReader()) - { - Format = reader.ReadByte(); - NumRelations = reader.ReadInt32(); + BinaryRobloxFile file = reader.File; - ChildrenIds = reader.ReadInstanceIds(NumRelations); - ParentIds = reader.ReadInstanceIds(NumRelations); - } - } + Format = reader.ReadByte(); + NumRelations = reader.ReadInt32(); - public void Assemble(BinaryRobloxFile file) - { + ChildrenIds = reader.ReadInstanceIds(NumRelations); + ParentIds = reader.ReadInstanceIds(NumRelations); + for (int i = 0; i < NumRelations; i++) { int childId = ChildrenIds[i]; @@ -31,5 +29,39 @@ child.Parent = (parentId >= 0 ? file.Instances[parentId] : file); } } + + public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + { + BinaryRobloxFile file = writer.File; + writer.StartWritingChunk(this); + + Format = 0; + NumRelations = file.Instances.Length; + + ChildrenIds = new List(); + ParentIds = new List(); + + foreach (Instance inst in file.Instances) + { + Instance parent = inst.Parent; + + int childId = int.Parse(inst.Referent); + int parentId = -1; + + if (parent != null) + parentId = int.Parse(parent.Referent); + + ChildrenIds.Add(childId); + ParentIds.Add(parentId); + } + + writer.Write(Format); + writer.Write(NumRelations); + + writer.WriteInstanceIds(ChildrenIds); + writer.WriteInstanceIds(ParentIds); + + return writer.FinishWritingChunk(); + } } } diff --git a/BinaryFormat/Chunks/PROP.cs b/BinaryFormat/Chunks/PROP.cs index 0912618..8f5c730 100644 --- a/BinaryFormat/Chunks/PROP.cs +++ b/BinaryFormat/Chunks/PROP.cs @@ -1,45 +1,37 @@ using System; -using System.IO; +using System.Collections.Generic; using System.Linq; +using System.Security.Cryptography; using RobloxFiles.Enums; using RobloxFiles.DataTypes; using RobloxFiles.Utility; +using System.Text; namespace RobloxFiles.BinaryFormat.Chunks { - public class PROP + public class PROP : IBinaryFileChunk { - public readonly string Name; - public readonly int TypeIndex; - public readonly PropertyType Type; + public string Name { get; internal set; } + public int TypeIndex { get; internal set; } - private BinaryRobloxFileReader Reader; - - public PROP(BinaryRobloxFileChunk chunk) + public PropertyType Type { get; internal set; } + public byte TypeId => (byte)Type; + + public void LoadFromReader(BinaryRobloxFileReader reader) { - Reader = chunk.GetDataReader(); + BinaryRobloxFile file = reader.File; - TypeIndex = Reader.ReadInt32(); - Name = Reader.ReadString(); + TypeIndex = reader.ReadInt32(); + Name = reader.ReadString(); - try - { - byte propType = Reader.ReadByte(); - Type = (PropertyType)propType; - } - catch - { - Type = PropertyType.Unknown; - } - } - - public void ReadProperties(BinaryRobloxFile file) - { + byte propType = reader.ReadByte(); + Type = (PropertyType)propType; + INST type = file.Types[TypeIndex]; Property[] props = new Property[type.NumInstances]; - int[] ids = type.InstanceIds; + var ids = type.InstanceIds; int instCount = type.NumInstances; for (int i = 0; i < instCount; i++) @@ -54,11 +46,10 @@ namespace RobloxFiles.BinaryFormat.Chunks } // Setup some short-hand functions for actions used during the read procedure. - var readInts = new Func(() => Reader.ReadInts(instCount)); - var readFloats = new Func(() => Reader.ReadFloats(instCount)); + var readInts = new Func(() => reader.ReadInts(instCount)); + var readFloats = new Func(() => reader.ReadFloats(instCount)); - - var loadProperties = new Action>(read => + var readProperties = new Action>(read => { for (int i = 0; i < instCount; i++) { @@ -71,13 +62,13 @@ namespace RobloxFiles.BinaryFormat.Chunks switch (Type) { case PropertyType.String: - loadProperties(i => + readProperties(i => { - string result = Reader.ReadString(); + string result = reader.ReadString(); // Leave an access point for the original byte sequence, in case this is a BinaryString. // This will allow the developer to read the sequence without any mangling from C# strings. - byte[] buffer = Reader.GetLastStringBuffer(); + byte[] buffer = reader.GetLastStringBuffer(); props[i].RawBuffer = buffer; return result; @@ -85,24 +76,24 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.Bool: - loadProperties(i => Reader.ReadBoolean()); + readProperties(i => reader.ReadBoolean()); break; case PropertyType.Int: int[] ints = readInts(); - loadProperties(i => ints[i]); + readProperties(i => ints[i]); break; case PropertyType.Float: float[] floats = readFloats(); - loadProperties(i => floats[i]); + readProperties(i => floats[i]); break; case PropertyType.Double: - loadProperties(i => Reader.ReadDouble()); + readProperties(i => reader.ReadDouble()); break; case PropertyType.UDim: float[] UDim_Scales = readFloats(); int[] UDim_Offsets = readInts(); - loadProperties(i => + readProperties(i => { float scale = UDim_Scales[i]; int offset = UDim_Offsets[i]; @@ -117,7 +108,7 @@ namespace RobloxFiles.BinaryFormat.Chunks int[] UDim2_Offsets_X = readInts(), UDim2_Offsets_Y = readInts(); - loadProperties(i => + readProperties(i => { float scaleX = UDim2_Scales_X[i], scaleY = UDim2_Scales_Y[i]; @@ -130,30 +121,35 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.Ray: - loadProperties(i => + readProperties(i => { - float[] rawOrigin = Reader.ReadFloats(3); - Vector3 origin = new Vector3(rawOrigin); + float posX = reader.ReadFloat(), + posY = reader.ReadFloat(), + posZ = reader.ReadFloat(); - float[] rawDirection = Reader.ReadFloats(3); - Vector3 direction = new Vector3(rawDirection); + float dirX = reader.ReadFloat(), + dirY = reader.ReadFloat(), + dirZ = reader.ReadFloat(); + + Vector3 origin = new Vector3(posX, posY, posZ); + Vector3 direction = new Vector3(dirX, dirY, dirZ); return new Ray(origin, direction); }); break; case PropertyType.Faces: - loadProperties(i => + readProperties(i => { - byte faces = Reader.ReadByte(); + byte faces = reader.ReadByte(); return (Faces)faces; }); break; case PropertyType.Axes: - loadProperties(i => + readProperties(i => { - byte axes = Reader.ReadByte(); + byte axes = reader.ReadByte(); return (Axes)axes; }); @@ -161,7 +157,7 @@ namespace RobloxFiles.BinaryFormat.Chunks case PropertyType.BrickColor: int[] BrickColorIds = readInts(); - loadProperties(i => + readProperties(i => { int number = BrickColorIds[i]; return BrickColor.FromNumber(number); @@ -173,7 +169,7 @@ namespace RobloxFiles.BinaryFormat.Chunks Color3_G = readFloats(), Color3_B = readFloats(); - loadProperties(i => + readProperties(i => { float r = Color3_R[i], g = Color3_G[i], @@ -187,7 +183,7 @@ namespace RobloxFiles.BinaryFormat.Chunks float[] Vector2_X = readFloats(), Vector2_Y = readFloats(); - loadProperties(i => + readProperties(i => { float x = Vector2_X[i], y = Vector2_Y[i]; @@ -201,7 +197,7 @@ namespace RobloxFiles.BinaryFormat.Chunks Vector3_Y = readFloats(), Vector3_Z = readFloats(); - loadProperties(i => + readProperties(i => { float x = Vector3_X[i], y = Vector3_Y[i], @@ -216,20 +212,20 @@ namespace RobloxFiles.BinaryFormat.Chunks // Temporarily load the rotation matrices into their properties. // We'll update them to CFrames once we iterate over the position data. - loadProperties(i => + readProperties(i => { - int normXY = Reader.ReadByte(); + byte b_OrientId = reader.ReadByte(); - if (normXY > 0) + if (b_OrientId > 0) { // Make sure this value is in a safe range. - normXY = (normXY - 1) % 36; + int orientId = (b_OrientId - 1) % 36; - NormalId normX = (NormalId)(normXY / 6); - Vector3 R0 = Vector3.FromNormalId(normX); + NormalId xColumn = (NormalId)(orientId / 6); + Vector3 R0 = Vector3.FromNormalId(xColumn); - NormalId normY = (NormalId)(normXY % 6); - Vector3 R1 = Vector3.FromNormalId(normY); + NormalId yColumn = (NormalId)(orientId % 6); + Vector3 R1 = Vector3.FromNormalId(yColumn); // Compute R2 using the cross product of R0 and R1. Vector3 R2 = R0.Cross(R1); @@ -244,8 +240,8 @@ namespace RobloxFiles.BinaryFormat.Chunks } else if (Type == PropertyType.Quaternion) { - float qx = Reader.ReadFloat(), qy = Reader.ReadFloat(), - qz = Reader.ReadFloat(), qw = Reader.ReadFloat(); + float qx = reader.ReadFloat(), qy = reader.ReadFloat(), + qz = reader.ReadFloat(), qw = reader.ReadFloat(); Quaternion quaternion = new Quaternion(qx, qy, qz, qw); var rotation = quaternion.ToCFrame(); @@ -258,7 +254,7 @@ namespace RobloxFiles.BinaryFormat.Chunks for (int m = 0; m < 9; m++) { - float value = Reader.ReadFloat(); + float value = reader.ReadFloat(); matrix[m] = value; } @@ -270,7 +266,7 @@ namespace RobloxFiles.BinaryFormat.Chunks CFrame_Y = readFloats(), CFrame_Z = readFloats(); - loadProperties(i => + readProperties(i => { float[] matrix = props[i].Value as float[]; @@ -289,14 +285,14 @@ namespace RobloxFiles.BinaryFormat.Chunks // TODO: I want to map these values to actual Roblox enums, but I'll have to add an // interpreter for the JSON API Dump to do it properly. - uint[] enums = Reader.ReadUInts(instCount); - loadProperties(i => enums[i]); + uint[] enums = reader.ReadUInts(instCount); + readProperties(i => enums[i]); break; case PropertyType.Ref: - int[] instIds = Reader.ReadInstanceIds(instCount); + var instIds = reader.ReadInstanceIds(instCount); - loadProperties(i => + readProperties(i => { int instId = instIds[i]; return instId >= 0 ? file.Instances[instId] : null; @@ -304,27 +300,27 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.Vector3int16: - loadProperties(i => + readProperties(i => { - short x = Reader.ReadInt16(), - y = Reader.ReadInt16(), - z = Reader.ReadInt16(); + short x = reader.ReadInt16(), + y = reader.ReadInt16(), + z = reader.ReadInt16(); return new Vector3int16(x, y, z); }); break; case PropertyType.NumberSequence: - loadProperties(i => + readProperties(i => { - int numKeys = Reader.ReadInt32(); + int numKeys = reader.ReadInt32(); var keypoints = new NumberSequenceKeypoint[numKeys]; for (int key = 0; key < numKeys; key++) { - float Time = Reader.ReadFloat(), - Value = Reader.ReadFloat(), - Envelope = Reader.ReadFloat(); + float Time = reader.ReadFloat(), + Value = reader.ReadFloat(), + Envelope = reader.ReadFloat(); keypoints[key] = new NumberSequenceKeypoint(Time, Value, Envelope); } @@ -334,20 +330,20 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.ColorSequence: - loadProperties(i => + readProperties(i => { - int numKeys = Reader.ReadInt32(); + int numKeys = reader.ReadInt32(); var keypoints = new ColorSequenceKeypoint[numKeys]; for (int key = 0; key < numKeys; key++) { - float Time = Reader.ReadFloat(), - R = Reader.ReadFloat(), - G = Reader.ReadFloat(), - B = Reader.ReadFloat(); + float Time = reader.ReadFloat(), + R = reader.ReadFloat(), + G = reader.ReadFloat(), + B = reader.ReadFloat(); Color3 Value = new Color3(R, G, B); - byte[] Reserved = Reader.ReadBytes(4); + byte[] Reserved = reader.ReadBytes(4); keypoints[key] = new ColorSequenceKeypoint(Time, Value, Reserved); } @@ -357,10 +353,10 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.NumberRange: - loadProperties(i => + readProperties(i => { - float min = Reader.ReadFloat(); - float max = Reader.ReadFloat(); + float min = reader.ReadFloat(); + float max = reader.ReadFloat(); return new NumberRange(min, max); }); @@ -370,7 +366,7 @@ namespace RobloxFiles.BinaryFormat.Chunks float[] Rect_X0 = readFloats(), Rect_Y0 = readFloats(), Rect_X1 = readFloats(), Rect_Y1 = readFloats(); - loadProperties(i => + readProperties(i => { float x0 = Rect_X0[i], y0 = Rect_Y0[i], x1 = Rect_X1[i], y1 = Rect_Y1[i]; @@ -380,17 +376,17 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.PhysicalProperties: - loadProperties(i => + readProperties(i => { - bool custom = Reader.ReadBoolean(); + bool custom = reader.ReadBoolean(); if (custom) { - float Density = Reader.ReadFloat(), - Friction = Reader.ReadFloat(), - Elasticity = Reader.ReadFloat(), - FrictionWeight = Reader.ReadFloat(), - ElasticityWeight = Reader.ReadFloat(); + float Density = reader.ReadFloat(), + Friction = reader.ReadFloat(), + Elasticity = reader.ReadFloat(), + FrictionWeight = reader.ReadFloat(), + ElasticityWeight = reader.ReadFloat(); return new PhysicalProperties ( @@ -407,11 +403,11 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.Color3uint8: - byte[] Color3uint8_R = Reader.ReadBytes(instCount), - Color3uint8_G = Reader.ReadBytes(instCount), - Color3uint8_B = Reader.ReadBytes(instCount); + byte[] Color3uint8_R = reader.ReadBytes(instCount), + Color3uint8_G = reader.ReadBytes(instCount), + Color3uint8_B = reader.ReadBytes(instCount); - loadProperties(i => + readProperties(i => { byte r = Color3uint8_R[i], g = Color3uint8_G[i], @@ -422,20 +418,20 @@ namespace RobloxFiles.BinaryFormat.Chunks break; case PropertyType.Int64: - long[] int64s = Reader.ReadInterleaved(instCount, (buffer, start) => + long[] Int64s = reader.ReadInterleaved(instCount, (buffer, start) => { long result = BitConverter.ToInt64(buffer, start); return (long)((ulong)result >> 1) ^ (-(result & 1)); }); - loadProperties(i => int64s[i]); + readProperties(i => Int64s[i]); break; case PropertyType.SharedString: - uint[] sharedKeys = Reader.ReadUInts(instCount); + uint[] SharedKeys = reader.ReadUInts(instCount); - loadProperties(i => + readProperties(i => { - uint key = sharedKeys[i]; + uint key = SharedKeys[i]; return file.SharedStrings[key]; }); @@ -443,9 +439,505 @@ namespace RobloxFiles.BinaryFormat.Chunks default: Console.WriteLine("Unhandled property type: {0}!", Type); break; + // } - Reader.Dispose(); + reader.Dispose(); + } + + internal static Dictionary CollectProperties(BinaryRobloxFileWriter writer, INST inst) + { + BinaryRobloxFile file = writer.File; + var propMap = new Dictionary(); + + foreach (int instId in inst.InstanceIds) + { + Instance instance = file.Instances[instId]; + + var props = instance.Properties; + var propNames = props.Keys; + + foreach (string propName in propNames) + { + if (!propMap.ContainsKey(propName)) + { + Property prop = props[propName]; + + PROP propChunk = new PROP() + { + Name = prop.Name, + Type = prop.Type, + TypeIndex = inst.TypeIndex + }; + + propMap.Add(propName, propChunk); + } + } + } + + return propMap; + } + + public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + { + BinaryRobloxFile file = writer.File; + + INST inst = file.Types[TypeIndex]; + var props = new List(); + + foreach (int instId in inst.InstanceIds) + { + Instance instance = file.Instances[instId]; + Property prop = instance.GetProperty(Name); + + if (prop == null) + throw new Exception($"Property {Name} must be defined in {instance.GetFullName()}!"); + else if (prop.Type != Type) + throw new Exception($"Property {Name} is not using the correct type in {instance.GetFullName()}!"); + + prop.CurrentWriter = writer; + props.Add(prop); + } + + writer.StartWritingChunk(this); + writer.Write(TypeIndex); + + writer.WriteString(Name); + writer.Write(TypeId); + + switch (Type) + { + case PropertyType.String: + props.ForEach(prop => + { + byte[] rawBuffer = prop.RawBuffer; + writer.Write(rawBuffer.Length); + writer.Write(rawBuffer); + }); + + break; + case PropertyType.Bool: + props.ForEach(prop => prop.WriteValue()); + break; + case PropertyType.Int: + var ints = new List(); + + props.ForEach(prop => + { + int value = prop.CastValue(); + ints.Add(value); + }); + + writer.WriteInts(ints); + break; + case PropertyType.Float: + var floats = new List(); + + props.ForEach(prop => + { + float value = prop.CastValue(); + floats.Add(value); + }); + + writer.WriteFloats(floats); + break; + case PropertyType.Double: + props.ForEach(prop => prop.WriteValue()); + break; + case PropertyType.UDim: + var UDim_Scales = new List(); + var UDim_Offsets = new List(); + + props.ForEach(prop => + { + UDim value = prop.CastValue(); + UDim_Scales.Add(value.Scale); + UDim_Offsets.Add(value.Offset); + }); + + writer.WriteFloats(UDim_Scales); + writer.WriteInts(UDim_Offsets); + + break; + case PropertyType.UDim2: + var UDim2_Scales_X = new List(); + var UDim2_Scales_Y = new List(); + + var UDim2_Offsets_X = new List(); + var UDim2_Offsets_Y = new List(); + + props.ForEach(prop => + { + UDim2 value = prop.CastValue(); + + UDim2_Scales_X.Add(value.X.Scale); + UDim2_Scales_Y.Add(value.Y.Scale); + + UDim2_Offsets_X.Add(value.X.Offset); + UDim2_Offsets_Y.Add(value.Y.Offset); + }); + + writer.WriteFloats(UDim2_Scales_X); + writer.WriteFloats(UDim2_Scales_Y); + + writer.WriteInts(UDim2_Offsets_X); + writer.WriteInts(UDim2_Offsets_Y); + + break; + case PropertyType.Ray: + props.ForEach(prop => + { + Ray ray = prop.CastValue(); + + Vector3 pos = ray.Origin; + writer.Write(pos.X); + writer.Write(pos.Y); + writer.Write(pos.Z); + + Vector3 dir = ray.Direction; + writer.Write(dir.X); + writer.Write(dir.Y); + writer.Write(dir.Z); + }); + + break; + case PropertyType.Faces: + case PropertyType.Axes: + props.ForEach(prop => prop.WriteValue()); + break; + case PropertyType.BrickColor: + var BrickColorIds = new List(); + + props.ForEach(prop => + { + BrickColor value = prop.CastValue(); + BrickColorIds.Add(value.Number); + }); + + writer.WriteInts(BrickColorIds); + break; + case PropertyType.Color3: + var Color3_R = new List(); + var Color3_G = new List(); + var Color3_B = new List(); + + props.ForEach(prop => + { + Color3 value = prop.CastValue(); + Color3_R.Add(value.R); + Color3_G.Add(value.G); + Color3_B.Add(value.B); + }); + + writer.WriteFloats(Color3_R); + writer.WriteFloats(Color3_G); + writer.WriteFloats(Color3_B); + + break; + case PropertyType.Vector2: + var Vector2_X = new List(); + var Vector2_Y = new List(); + + props.ForEach(prop => + { + Vector2 value = prop.CastValue(); + Vector2_X.Add(value.X); + Vector2_Y.Add(value.Y); + }); + + writer.WriteFloats(Vector2_X); + writer.WriteFloats(Vector2_Y); + + break; + case PropertyType.Vector3: + var Vector3_X = new List(); + var Vector3_Y = new List(); + var Vector3_Z = new List(); + + props.ForEach(prop => + { + Vector3 value = prop.CastValue(); + Vector3_X.Add(value.X); + Vector3_Y.Add(value.Y); + Vector3_Z.Add(value.Z); + }); + + writer.WriteFloats(Vector3_X); + writer.WriteFloats(Vector3_Y); + writer.WriteFloats(Vector3_Z); + + break; + case PropertyType.CFrame: + case PropertyType.Quaternion: + var CFrame_X = new List(); + var CFrame_Y = new List(); + var CFrame_Z = new List(); + + props.ForEach(prop => + { + CFrame value = null; + + if (prop.Value is Quaternion) + { + Quaternion q = prop.CastValue(); + value = q.ToCFrame(); + } + else + { + value = prop.CastValue(); + } + + Vector3 pos = value.Position; + CFrame_X.Add(pos.X); + CFrame_Y.Add(pos.Y); + CFrame_Z.Add(pos.Z); + + int orientId = value.GetOrientId(); + writer.Write((byte)(orientId + 1)); + + if (orientId == -1) + { + if (Type == PropertyType.Quaternion) + { + Quaternion quat = new Quaternion(value); + writer.Write(quat.X); + writer.Write(quat.Y); + writer.Write(quat.Z); + writer.Write(quat.W); + } + else + { + float[] components = value.GetComponents(); + + for (int i = 3; i < 12; i++) + { + float component = components[i]; + writer.Write(component); + } + } + } + }); + + writer.WriteFloats(CFrame_X); + writer.WriteFloats(CFrame_Y); + writer.WriteFloats(CFrame_Z); + + break; + case PropertyType.Enum: + var Enums = new List(); + + props.ForEach(prop => + { + uint value = prop.CastValue(); + Enums.Add(value); + }); + + writer.WriteInterleaved(Enums); + break; + case PropertyType.Ref: + var InstanceIds = new List(); + + props.ForEach(prop => + { + int referent = -1; + + if (prop.Value != null) + { + Instance value = prop.CastValue(); + referent = int.Parse(value.Referent); + } + + InstanceIds.Add(referent); + }); + + writer.WriteInstanceIds(InstanceIds); + break; + case PropertyType.Vector3int16: + props.ForEach(prop => + { + Vector3int16 value = prop.CastValue(); + writer.Write(value.X); + writer.Write(value.Y); + writer.Write(value.Z); + }); + + break; + case PropertyType.NumberSequence: + props.ForEach(prop => + { + NumberSequence value = prop.CastValue(); + + var keyPoints = value.Keypoints; + writer.Write(keyPoints.Length); + + foreach (var keyPoint in keyPoints) + { + writer.Write(keyPoint.Time); + writer.Write(keyPoint.Value); + writer.Write(keyPoint.Envelope); + } + }); + + break; + case PropertyType.ColorSequence: + props.ForEach(prop => + { + ColorSequence value = prop.CastValue(); + + var keyPoints = value.Keypoints; + writer.Write(keyPoints.Length); + + foreach (var keyPoint in keyPoints) + { + Color3 color = keyPoint.Value; + writer.Write(keyPoint.Time); + + writer.Write(color.R); + writer.Write(color.G); + writer.Write(color.B); + + writer.Write(0); + } + }); + + break; + case PropertyType.NumberRange: + props.ForEach(prop => + { + NumberRange value = prop.CastValue(); + writer.Write(value.Min); + writer.Write(value.Max); + }); + + break; + case PropertyType.Rect: + var Rect_X0 = new List(); + var Rect_Y0 = new List(); + + var Rect_X1 = new List(); + var Rect_Y1 = new List(); + + props.ForEach(prop => + { + Rect value = prop.CastValue(); + + Vector2 min = value.Min; + Rect_X0.Add(min.X); + Rect_Y0.Add(min.Y); + + Vector2 max = value.Max; + Rect_X1.Add(max.X); + Rect_Y1.Add(max.Y); + }); + + writer.WriteFloats(Rect_X0); + writer.WriteFloats(Rect_Y0); + + writer.WriteFloats(Rect_X1); + writer.WriteFloats(Rect_Y1); + + break; + case PropertyType.PhysicalProperties: + props.ForEach(prop => + { + bool custom = (prop.Value != null); + writer.Write(custom); + + if (custom) + { + PhysicalProperties value = prop.CastValue(); + + writer.Write(value.Density); + writer.Write(value.Friction); + writer.Write(value.Elasticity); + + writer.Write(value.FrictionWeight); + writer.Write(value.ElasticityWeight); + } + }); + + break; + case PropertyType.Color3uint8: + var Color3uint8_R = new List(); + var Color3uint8_G = new List(); + var Color3uint8_B = new List(); + + props.ForEach(prop => + { + Color3 value = prop.CastValue(); + + byte r = (byte)(value.R * 255); + Color3uint8_R.Add(r); + + byte g = (byte)(value.G * 255); + Color3uint8_G.Add(g); + + byte b = (byte)(value.B * 255); + Color3uint8_B.Add(b); + }); + + writer.Write(Color3uint8_R.ToArray()); + writer.Write(Color3uint8_G.ToArray()); + writer.Write(Color3uint8_B.ToArray()); + + break; + case PropertyType.Int64: + var Int64s = new List(); + + props.ForEach(prop => + { + long value = prop.CastValue(); + Int64s.Add(value); + }); + + writer.WriteInterleaved(Int64s, value => + { + // Move the sign bit to the front. + return (value << 1) ^ (value >> 63); + }); + + break; + case PropertyType.SharedString: + var sharedKeys = new List(); + SSTR sstr = file.SSTR; + + if (sstr == null) + { + sstr = new SSTR(); + file.SSTR = sstr; + } + + props.ForEach(prop => + { + uint sharedKey = 0; + + string value = prop.CastValue(); + byte[] buffer = Encoding.UTF8.GetBytes(value); + + using (MD5 md5 = MD5.Create()) + { + byte[] hash = md5.ComputeHash(buffer); + string key = Convert.ToBase64String(hash); + + if (!sstr.Lookup.ContainsKey(key)) + { + uint id = (uint)(sstr.NumHashes++); + sstr.Strings.Add(id, value); + sstr.Lookup.Add(key, id); + } + + sharedKey = sstr.Lookup[key]; + } + + sharedKeys.Add(sharedKey); + }); + + writer.WriteInterleaved(sharedKeys); + break; + // + } + + return writer.FinishWritingChunk(); } } } diff --git a/BinaryFormat/Chunks/SSTR.cs b/BinaryFormat/Chunks/SSTR.cs index 46aa6e5..87da5bb 100644 --- a/BinaryFormat/Chunks/SSTR.cs +++ b/BinaryFormat/Chunks/SSTR.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; namespace RobloxFiles.BinaryFormat.Chunks { - public class SSTR + public class SSTR : IBinaryFileChunk { public int Version; public int NumHashes; @@ -11,27 +11,50 @@ namespace RobloxFiles.BinaryFormat.Chunks public Dictionary Lookup = new Dictionary(); public Dictionary Strings = new Dictionary(); - public SSTR(BinaryRobloxFileChunk chunk) + public void LoadFromReader(BinaryRobloxFileReader reader) { - using (BinaryRobloxFileReader reader = chunk.GetDataReader()) + BinaryRobloxFile file = reader.File; + + Version = reader.ReadInt32(); + NumHashes = reader.ReadInt32(); + + for (uint id = 0; id < NumHashes; id++) { - Version = reader.ReadInt32(); - NumHashes = reader.ReadInt32(); + byte[] md5 = reader.ReadBytes(16); - for (uint id = 0; id < NumHashes; id++) - { - byte[] md5 = reader.ReadBytes(16); + int length = reader.ReadInt32(); + byte[] data = reader.ReadBytes(length); - int length = reader.ReadInt32(); - byte[] data = reader.ReadBytes(length); + string key = Convert.ToBase64String(md5); + string value = Convert.ToBase64String(data); - string key = Convert.ToBase64String(md5); - string value = Convert.ToBase64String(data); - - Lookup.Add(key, id); - Strings.Add(id, value); - } + Lookup.Add(key, id); + Strings.Add(id, value); } + + file.SSTR = this; + } + + public BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer) + { + writer.StartWritingChunk(this); + + writer.Write(Version); + writer.Write(NumHashes); + + foreach (var pair in Lookup) + { + string key = pair.Key; + byte[] md5 = Convert.FromBase64String(key); + + uint id = pair.Value; + string value = Strings[id]; + + writer.Write(md5); + writer.WriteString(value); + } + + return writer.FinishWritingChunk(); } } } diff --git a/BinaryFormat/BinaryFileReader.cs b/BinaryFormat/IO/BinaryFileReader.cs similarity index 90% rename from BinaryFormat/BinaryFileReader.cs rename to BinaryFormat/IO/BinaryFileReader.cs index e6de267..5c0cbde 100644 --- a/BinaryFormat/BinaryFileReader.cs +++ b/BinaryFormat/IO/BinaryFileReader.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Runtime.InteropServices; using System.Text; @@ -7,9 +9,14 @@ namespace RobloxFiles.BinaryFormat { public class BinaryRobloxFileReader : BinaryReader { - public BinaryRobloxFileReader(Stream stream) : base(stream) { } + public readonly BinaryRobloxFile File; private byte[] lastStringBuffer = new byte[0] { }; + public BinaryRobloxFileReader(BinaryRobloxFile file, Stream stream) : base(stream) + { + File = file; + } + // Reads 'count * sizeof(T)' interleaved bytes and converts // them into an array of T[count] where each value in the // array has been decoded by the provided 'decode' function. @@ -74,14 +81,14 @@ namespace RobloxFiles.BinaryFormat } // Reads and accumulates an interleaved buffer of integers. - public int[] ReadInstanceIds(int count) + public List ReadInstanceIds(int count) { int[] values = ReadInts(count); for (int i = 1; i < count; ++i) values[i] += values[i - 1]; - return values; + return values.ToList(); } public override string ReadString() diff --git a/BinaryFormat/IO/BinaryFileWriter.cs b/BinaryFormat/IO/BinaryFileWriter.cs new file mode 100644 index 0000000..6c6efbc --- /dev/null +++ b/BinaryFormat/IO/BinaryFileWriter.cs @@ -0,0 +1,240 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; + +using RobloxFiles.BinaryFormat.Chunks; + +namespace RobloxFiles.BinaryFormat +{ + public class BinaryRobloxFileWriter : BinaryWriter + { + public IBinaryFileChunk Chunk { get; private set; } + public bool WritingChunk { get; private set; } + + public string ChunkType { get; private set; } + public long ChunkStart { get; private set; } + + public Dictionary TypeMap; + public readonly BinaryRobloxFile File; + public List Instances; + + public BinaryRobloxFileWriter(BinaryRobloxFile file, Stream workBuffer = null) : base(workBuffer ?? new MemoryStream()) + { + File = file; + + ChunkStart = 0; + ChunkType = ""; + + Instances = new List(); + TypeMap = new Dictionary(); + } + + private static byte[] GetBytes(T value, int bufferSize, IntPtr converter) + { + byte[] bytes = new byte[bufferSize]; + + Marshal.StructureToPtr(value, converter, true); + Marshal.Copy(converter, bytes, 0, bufferSize); + + return bytes; + } + + public static byte[] GetBytes(T value) where T : struct + { + int bufferSize = Marshal.SizeOf(); + IntPtr converter = Marshal.AllocHGlobal(bufferSize); + + var result = GetBytes(value, bufferSize, converter); + Marshal.FreeHGlobal(converter); + + return result; + } + + // Writes 'count * sizeof(T)' interleaved bytes from a List of T where + // each value in the array will be encoded using the provided 'encode' function. + public void WriteInterleaved(List values, Func encode = null) where T : struct + { + int count = values.Count; + int bufferSize = Marshal.SizeOf(); + + byte[][] blocks = new byte[count][]; + IntPtr converter = Marshal.AllocHGlobal(bufferSize); + + for (int i = 0; i < count; i++) + { + T value = values[i]; + + if (encode != null) + value = encode(value); + + blocks[i] = GetBytes(value, bufferSize, converter); + } + + for (int layer = bufferSize - 1; layer >= 0; layer--) + { + for (int i = 0; i < count; i++) + { + byte value = blocks[i][layer]; + Write(value); + } + } + + Marshal.FreeHGlobal(converter); + } + + // Encodes an int for an interleaved buffer. + private int EncodeInt(int value) + { + return (value << 1) ^ (value >> 31); + } + + // Encodes a float for an interleaved buffer. + private float EncodeFloat(float value) + { + byte[] buffer = BitConverter.GetBytes(value); + uint bits = BitConverter.ToUInt32(buffer, 0); + + bits = (bits << 1) | (bits >> 31); + buffer = BitConverter.GetBytes(bits); + + return BitConverter.ToSingle(buffer, 0); + } + + // Writes an interleaved list of ints. + public void WriteInts(List values) + { + WriteInterleaved(values, EncodeInt); + } + + // Writes an interleaved list of floats + public void WriteFloats(List values) + { + WriteInterleaved(values, EncodeFloat); + } + + // Writes an accumlated array of integers. + public void WriteInstanceIds(List values) + { + int numIds = values.Count; + var instIds = new List(values); + + for (int i = 1; i < numIds; ++i) + instIds[i] -= values[i - 1];; + + WriteInts(instIds); + } + + // Writes a string to the buffer with the option to exclude a length prefix. + public void WriteString(string value, bool raw = false) + { + byte[] buffer = Encoding.UTF8.GetBytes(value); + + if (!raw) + { + int length = buffer.Length; + Write(length); + } + + Write(buffer); + } + + internal void RecordInstances(IEnumerable instances) + { + foreach (Instance instance in instances) + { + int instId = (int)(File.NumInstances++); + + instance.Referent = instId.ToString(); + Instances.Add(instance); + + string className = instance.ClassName; + INST inst = null; + + if (!TypeMap.ContainsKey(className)) + { + inst = new INST() + { + TypeName = className, + InstanceIds = new List(), + IsService = instance.IsService + }; + + TypeMap.Add(className, inst); + } + else + { + inst = TypeMap[className]; + } + + inst.NumInstances++; + inst.InstanceIds.Add(instId); + + RecordInstances(instance.GetChildren()); + } + } + + // Marks that we are writing a chunk. + public bool StartWritingChunk(string chunkType) + { + if (chunkType.Length != 4) + throw new Exception("BinaryFileWriter.StartWritingChunk - ChunkType length should be 4!"); + + if (!WritingChunk) + { + WritingChunk = true; + + ChunkType = chunkType; + ChunkStart = BaseStream.Position; + + return true; + } + + return false; + } + + // Marks that we are writing a chunk. + public bool StartWritingChunk(IBinaryFileChunk chunk) + { + if (!WritingChunk) + { + string chunkType = chunk.GetType().Name; + + StartWritingChunk(chunkType); + Chunk = chunk; + + return true; + } + + return false; + } + + // Compresses the data that was written into a BinaryRobloxFileChunk and writes it. + public BinaryRobloxFileChunk FinishWritingChunk(bool compress = true) + { + if (!WritingChunk) + throw new Exception($"BinaryRobloxFileWriter: Cannot finish writing a chunk without starting one!"); + + // Create the compressed chunk. + var chunk = new BinaryRobloxFileChunk(this, compress); + + // Clear out the uncompressed data. + BaseStream.Position = ChunkStart; + BaseStream.SetLength(ChunkStart); + + // Write the compressed chunk. + chunk.Handler = Chunk; + chunk.WriteChunk(this); + + // Clean up for the next chunk. + WritingChunk = false; + + ChunkStart = 0; + ChunkType = ""; + Chunk = null; + + return chunk; + } + } +} diff --git a/DataTypes/CFrame.cs b/DataTypes/CFrame.cs index 8545d9c..bd4295b 100644 --- a/DataTypes/CFrame.cs +++ b/DataTypes/CFrame.cs @@ -335,5 +335,55 @@ namespace RobloxFiles.DataTypes return new float[] { x, y, z }; } + + public bool IsAxisAligned() + { + float[] matrix = GetComponents(); + + byte sum0 = 0, + sum1 = 0; + + for (int i = 3; i < 12; i++) + { + float t = matrix[i]; + + if (Math.Abs(t - 1f) < 10e-5f) + { + // Approximately ±1 + sum1++; + } + else if (Math.Abs(t) < 10e-5f) + { + // Approximately ±0 + sum0++; + } + } + + return (sum0 == 6 && sum1 == 3); + } + + private static bool IsLegalOrientId(int orientId) + { + int xNormalAbs = (orientId / 6) % 3; + int yNormalAbs = orientId % 3; + + return (xNormalAbs != yNormalAbs); + } + + public int GetOrientId() + { + if (!IsAxisAligned()) + return -1; + + int xNormal = RightVector.ToNormalId(); + int yNormal = UpVector.ToNormalId(); + + int orientId = (6 * xNormal) + yNormal; + + if (!IsLegalOrientId(orientId)) + return -1; + + return orientId; + } } } diff --git a/DataTypes/Vector3.cs b/DataTypes/Vector3.cs index b8f6a62..7aa0d76 100644 --- a/DataTypes/Vector3.cs +++ b/DataTypes/Vector3.cs @@ -125,9 +125,30 @@ namespace RobloxFiles.DataTypes return this + (other - this) * t; } - public bool isClose(Vector3 other, float epsilon = 0.0f) + public bool IsClose(Vector3 other, float epsilon = 0.0f) { return (other - this).Magnitude <= Math.Abs(epsilon); } + + public int ToNormalId() + { + int result = -1; + + for (int i = 0; i < 6; i++) + { + NormalId normalId = (NormalId)i; + Vector3 normal = FromNormalId(normalId); + + float dotProd = normal.Dot(this); + + if (Math.Abs(dotProd - 1f) < 10e-5f) + { + result = i; + break; + } + } + + return result; + } } } diff --git a/Interfaces/IBinaryFileChunk.cs b/Interfaces/IBinaryFileChunk.cs new file mode 100644 index 0000000..6011ecc --- /dev/null +++ b/Interfaces/IBinaryFileChunk.cs @@ -0,0 +1,8 @@ +namespace RobloxFiles.BinaryFormat +{ + public interface IBinaryFileChunk + { + void LoadFromReader(BinaryRobloxFileReader reader); + BinaryRobloxFileChunk SaveAsChunk(BinaryRobloxFileWriter writer); + } +} diff --git a/XmlFormat/PropertyTokens/IXmlPropertyToken.cs b/Interfaces/IXmlPropertyToken.cs similarity index 100% rename from XmlFormat/PropertyTokens/IXmlPropertyToken.cs rename to Interfaces/IXmlPropertyToken.cs diff --git a/RobloxFile.cs b/RobloxFile.cs index eb3a4c6..2e8d6b6 100644 --- a/RobloxFile.cs +++ b/RobloxFile.cs @@ -10,7 +10,7 @@ namespace RobloxFiles { /// /// Represents a loaded *.rbxl/*.rbxm Roblox file. - /// All of the surface-level Instances are stored in the RobloxFile's 'Contents' property. + /// The contents of the RobloxFile are stored as its children. /// public abstract class RobloxFile : Instance { diff --git a/RobloxFileFormat.csproj b/RobloxFileFormat.csproj index d822947..05f162e 100644 --- a/RobloxFileFormat.csproj +++ b/RobloxFileFormat.csproj @@ -63,13 +63,15 @@ - + + + @@ -88,7 +90,7 @@ - + @@ -146,5 +148,6 @@ false + \ No newline at end of file diff --git a/Tree/Instance.cs b/Tree/Instance.cs index 4ec8cf8..e282d6e 100644 --- a/Tree/Instance.cs +++ b/Tree/Instance.cs @@ -18,17 +18,23 @@ namespace RobloxFiles private Dictionary props = new Dictionary(); public IReadOnlyDictionary Properties => props; - private List Children = new List(); + protected List Children = new List(); private Instance parent; /// The name of this Instance, if a Name property is defined. public override string ToString() => Name; - /// A unique identifier for this instance when being serialized as XML. - public string XmlReferent { get; internal set; } + /// A unique identifier for this instance when being serialized. + public string Referent { get; internal set; } /// Indicates whether the parent of this object is locked. - public bool ParentLocked { get; protected set; } + public bool ParentLocked { get; internal set; } + + /// Indicates whether this Instance is marked as a Service in the binary file format. + public bool IsService { get; internal set; } + + /// If this instance is a service, this indicates whether the service should be loaded via GetService when Roblox loads the place file. + public bool IsRootedService { get; internal set; } /// Indicates whether this object should be serialized. public bool Archivable = true; diff --git a/Tree/Property.cs b/Tree/Property.cs index f422573..a7807bf 100644 --- a/Tree/Property.cs +++ b/Tree/Property.cs @@ -1,4 +1,6 @@ using System; + +using RobloxFiles.BinaryFormat; using RobloxFiles.BinaryFormat.Chunks; namespace RobloxFiles @@ -47,6 +49,8 @@ namespace RobloxFiles public string XmlToken = ""; public byte[] RawBuffer { get; internal set; } + internal BinaryRobloxFileWriter CurrentWriter; + public bool HasRawBuffer { get @@ -83,10 +87,9 @@ namespace RobloxFiles public Property(string name = "", PropertyType type = PropertyType.Unknown, Instance instance = null) { + Instance = instance; Name = name; Type = type; - - Instance = instance; } public Property(Instance instance, PROP property) @@ -116,5 +119,28 @@ namespace RobloxFiles return string.Join(" ", typeName, Name, '=', valueLabel); } + + public T CastValue() + { + T result; + + if (Value is T) + result = (T)Value; + else + result = default(T); + + return result; + } + + internal void WriteValue() where T : struct + { + if (CurrentWriter == null) + throw new Exception("Property.CurrentWriter must be set to use WriteValue"); + + T value = CastValue(); + + byte[] bytes = BinaryRobloxFileWriter.GetBytes(value); + CurrentWriter.Write(bytes); + } } } \ No newline at end of file diff --git a/XmlFormat/IO/XmlFileReader.cs b/XmlFormat/IO/XmlFileReader.cs index 80fd12d..bc5b8ed 100644 --- a/XmlFormat/IO/XmlFileReader.cs +++ b/XmlFormat/IO/XmlFileReader.cs @@ -98,7 +98,7 @@ namespace RobloxFiles.XmlFormat if (refToken != null && file != null) { string referent = refToken.InnerText; - inst.XmlReferent = referent; + inst.Referent = referent; if (file.Instances.ContainsKey(referent)) throw error("Got an Item with a duplicate 'referent' attribute!"); diff --git a/XmlFormat/IO/XmlFileWriter.cs b/XmlFormat/IO/XmlFileWriter.cs index 3ca5c00..e8bec1d 100644 --- a/XmlFormat/IO/XmlFileWriter.cs +++ b/XmlFormat/IO/XmlFileWriter.cs @@ -40,10 +40,10 @@ namespace RobloxFiles.XmlFormat foreach (Instance child in inst.GetChildren()) RecordInstances(file, child); - if (inst.XmlReferent.Length < 35) - inst.XmlReferent = CreateReferent(); + if (inst.Referent.Length < 35) + inst.Referent = CreateReferent(); - file.Instances.Add(inst.XmlReferent, inst); + file.Instances.Add(inst.Referent, inst); } public static XmlElement CreateRobloxElement(XmlDocument doc) @@ -141,7 +141,7 @@ namespace RobloxFiles.XmlFormat { XmlElement instNode = doc.CreateElement("Item"); instNode.SetAttribute("class", instance.ClassName); - instNode.SetAttribute("referent", instance.XmlReferent); + instNode.SetAttribute("referent", instance.Referent); XmlElement propsNode = doc.CreateElement("Properties"); instNode.AppendChild(propsNode); diff --git a/XmlFormat/PropertyTokens/Tokens/Ref.cs b/XmlFormat/PropertyTokens/Tokens/Ref.cs index 3b959d6..05c46dd 100644 --- a/XmlFormat/PropertyTokens/Tokens/Ref.cs +++ b/XmlFormat/PropertyTokens/Tokens/Ref.cs @@ -22,7 +22,7 @@ namespace RobloxFiles.XmlFormat.PropertyTokens if (prop.Value != null && prop.Value.ToString() != "null") { Instance inst = prop.Value as Instance; - result = inst.XmlReferent; + result = inst.Referent; } node.InnerText = result; diff --git a/XmlFormat/PropertyTokens/XmlPropertyTokens.cs b/XmlFormat/PropertyTokens/XmlPropertyTokens.cs index 4e31090..1bc1f6e 100644 --- a/XmlFormat/PropertyTokens/XmlPropertyTokens.cs +++ b/XmlFormat/PropertyTokens/XmlPropertyTokens.cs @@ -40,12 +40,13 @@ namespace RobloxFiles.XmlFormat try { string value = token.InnerText; + Type type = typeof(T); - if (typeof(T) == typeof(int)) + if (type == typeof(int)) prop.Value = Formatting.ParseInt(value); - else if (typeof(T) == typeof(float)) + else if (type == typeof(float)) prop.Value = Formatting.ParseFloat(value); - else if (typeof(T) == typeof(double)) + else if (type == typeof(double)) prop.Value = Formatting.ParseDouble(value); if (prop.Value == null) diff --git a/XmlFormat/XmlRobloxFile.cs b/XmlFormat/XmlRobloxFile.cs index 9581e51..33763a7 100644 --- a/XmlFormat/XmlRobloxFile.cs +++ b/XmlFormat/XmlRobloxFile.cs @@ -29,7 +29,7 @@ namespace RobloxFiles.XmlFormat string xml = Encoding.UTF8.GetString(buffer); Root.LoadXml(xml); } - catch (Exception e) + catch { throw new Exception("XmlRobloxFile: Could not read provided buffer as XML!"); } @@ -110,7 +110,7 @@ namespace RobloxFiles.XmlFormat } else { - throw new Exception("XmlRobloxFile: No 'roblox' tag found!"); + throw new Exception("XmlRobloxFile: No 'roblox' element found!"); } }