Skip to content

Commit

Permalink
LSF version number encoding fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Norbyte
Norbyte committed Aug 21, 2021
1 parent e59168f commit 14a17db
Show file tree
Hide file tree
Showing 7 changed files with 129 additions and 101 deletions.
1 change: 1 addition & 0 deletions ConverterApp/OsirisPane.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ private void SaveSavegameDatabase()
var rewrittenStream = new MemoryStream();
var rsrcWriter = new LSFWriter(rewrittenStream);
rsrcWriter.Version = conversionParams.LSF;
rsrcWriter.EncodeSiblingData = false;
rsrcWriter.Write(resource);
rewrittenStream.Seek(0, SeekOrigin.Begin);

Expand Down
18 changes: 17 additions & 1 deletion LSLib/LS/Resource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public static PackedVersion FromInt64(Int64 packed)
Major = (UInt32)((packed >> 55) & 0x7f),
Minor = (UInt32)((packed >> 47) & 0xff),
Revision = (UInt32)((packed >> 31) & 0xffff),
Build = (UInt32)(packed & 0x7FFFFFFF),
Build = (UInt32)(packed & 0x7fffffff),
};
}

Expand All @@ -40,6 +40,22 @@ public static PackedVersion FromInt32(Int32 packed)
Build = (UInt32)(packed & 0xffff),
};
}

public Int32 ToVersion32()
{
return (Int32)((Major & 0x0f) << 28 |
(Minor & 0x0f) << 24 |
(Revision & 0xff) << 16 |
(Build & 0xffff) << 0);
}

public Int64 ToVersion64()
{
return (Int64)(((Int64)Major & 0x7f) << 55 |
((Int64)Minor & 0xff) << 47 |
((Int64)Revision & 0xffff) << 31 |
((Int64)Build & 0x7fffffff) << 0);
}
}

[StructLayout(LayoutKind.Sequential)]
Expand Down
43 changes: 24 additions & 19 deletions LSLib/LS/Resources/LSF/LSFCommon.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
namespace LSLib.LS
{
[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct Header
internal struct LSFMagic
{
/// <summary>
/// LSOF file signature
Expand All @@ -15,18 +15,33 @@ internal struct Header
/// LSOF file signature; should be the same as LSFHeader.Signature
/// </summary>
public UInt32 Magic;

/// <summary>
/// Version of the LSOF file; D:OS EE is version 1/2, D:OS 2 is version 3
/// </summary>
public UInt32 Version;
};

[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct LSFHeader
{
/// <summary>
/// Possibly version number? (major, minor, rev, build)
/// </summary>
public Int32 EngineVersion;
};

[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct LSFHeaderV5
{
/// <summary>
/// Possibly version number? (major, minor, rev, build)
/// </summary>
public UInt32 EngineVersion;
public Int64 EngineVersion;
};

[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct Metadata
internal struct LSFMetadata
{
/// <summary>
/// Total uncompressed size of the string hash table
Expand Down Expand Up @@ -76,21 +91,11 @@ internal struct Metadata
public UInt32 HasSiblingData;
}

[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct MetadataV5
{
/// <summary>
/// Unknown, always 0x02000002
/// </summary>
public UInt32 Unknown;
public Metadata Metadata;
}

/// <summary>
/// Node (structure) entry in the LSF file
/// </summary>
[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct NodeEntryV2
internal struct LSFNodeEntryV2
{
/// <summary>
/// Name of this node
Expand Down Expand Up @@ -129,7 +134,7 @@ public int NameOffset
/// Node (structure) entry in the LSF file
/// </summary>
[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct NodeEntryV3
internal struct LSFNodeEntryV3
{
/// <summary>
/// Name of this node
Expand Down Expand Up @@ -172,7 +177,7 @@ public int NameOffset
/// <summary>
/// Processed node information for a node in the LSF file
/// </summary>
internal class NodeInfo
internal class LSFNodeInfo
{
/// <summary>
/// Index of the parent node
Expand All @@ -198,7 +203,7 @@ internal class NodeInfo
/// V2 attribute extension in the LSF file
/// </summary>
[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct AttributeEntryV2
internal struct LSFAttributeEntryV2
{
/// <summary>
/// Name of this attribute
Expand Down Expand Up @@ -255,7 +260,7 @@ public uint Length
/// V3 attribute extension in the LSF file
/// </summary>
[StructLayout(LayoutKind.Sequential, Pack = 1)]
internal struct AttributeEntryV3
internal struct LSFAttributeEntryV3
{
/// <summary>
/// Name of this attribute
Expand Down Expand Up @@ -313,7 +318,7 @@ public uint Length
}
};

internal class AttributeInfo
internal class LSFAttributeInfo
{
/// <summary>
/// Index into name hash table
Expand Down
80 changes: 43 additions & 37 deletions LSLib/LS/Resources/LSF/LSFReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ public class LSFReader : IDisposable
/// <summary>
/// Preprocessed list of nodes (structures)
/// </summary>
private List<NodeInfo> Nodes;
private List<LSFNodeInfo> Nodes;
/// <summary>
/// Preprocessed list of node attributes
/// </summary>
private List<AttributeInfo> Attributes;
private List<LSFAttributeInfo> Attributes;
/// <summary>
/// Node instances
/// </summary>
Expand All @@ -39,7 +39,11 @@ public class LSFReader : IDisposable
/// <summary>
/// Version of the file we're serializing
/// </summary>
private uint Version;
private LSFVersion Version;
/// <summary>
/// Game version that generated the LSF file
/// </summary>
private PackedVersion GameVersion;

public LSFReader(Stream stream)
{
Expand Down Expand Up @@ -106,22 +110,22 @@ private void ReadNodes(Stream s, bool longNodes)
Int32 index = 0;
while (s.Position < s.Length)
{
var resolved = new NodeInfo();
var resolved = new LSFNodeInfo();
#if DEBUG_LSF_SERIALIZATION
var pos = s.Position;
#endif

if (longNodes)
{
var item = BinUtils.ReadStruct<NodeEntryV3>(reader);
var item = BinUtils.ReadStruct<LSFNodeEntryV3>(reader);
resolved.ParentIndex = item.ParentIndex;
resolved.NameIndex = item.NameIndex;
resolved.NameOffset = item.NameOffset;
resolved.FirstAttributeIndex = item.FirstAttributeIndex;
}
else
{
var item = BinUtils.ReadStruct<NodeEntryV2>(reader);
var item = BinUtils.ReadStruct<LSFNodeEntryV2>(reader);
resolved.ParentIndex = item.ParentIndex;
resolved.NameIndex = item.NameIndex;
resolved.NameOffset = item.NameOffset;
Expand Down Expand Up @@ -159,9 +163,9 @@ private void ReadAttributesV2(Stream s)
Int32 index = 0;
while (s.Position < s.Length)
{
var attribute = BinUtils.ReadStruct<AttributeEntryV2>(reader);
var attribute = BinUtils.ReadStruct<LSFAttributeEntryV2>(reader);

var resolved = new AttributeInfo();
var resolved = new LSFAttributeInfo();
resolved.NameIndex = attribute.NameIndex;
resolved.NameOffset = attribute.NameOffset;
resolved.TypeId = attribute.TypeId;
Expand Down Expand Up @@ -233,9 +237,9 @@ private void ReadAttributesV3(Stream s)
{
while (s.Position < s.Length)
{
var attribute = BinUtils.ReadStruct<AttributeEntryV3>(reader);
var attribute = BinUtils.ReadStruct<LSFAttributeEntryV3>(reader);

var resolved = new AttributeInfo();
var resolved = new LSFAttributeInfo();
resolved.NameIndex = attribute.NameIndex;
resolved.NameOffset = attribute.NameOffset;
resolved.TypeId = attribute.TypeId;
Expand Down Expand Up @@ -263,9 +267,9 @@ private void ReadAttributesV3(Stream s)
}
}

private byte[] Decompress(BinaryReader reader, uint compressedSize, uint uncompressedSize, Header header, Metadata metadata)
private byte[] Decompress(BinaryReader reader, uint compressedSize, uint uncompressedSize, LSFMetadata metadata)
{
bool chunked = header.Version >= (ulong) LSFVersion.VerChunkedCompress;
bool chunked = Version >= LSFVersion.VerChunkedCompress;
byte[] compressed = reader.ReadBytes((int)compressedSize);
return BinUtils.Decompress(compressed, (int)uncompressedSize, metadata.CompressionFlags, chunked);
}
Expand All @@ -274,35 +278,37 @@ public Resource Read()
{
using (var reader = new BinaryReader(Stream))
{
var hdr = BinUtils.ReadStruct<Header>(reader);
if (hdr.Magic != BitConverter.ToUInt32(Header.Signature, 0))
var magic = BinUtils.ReadStruct<LSFMagic>(reader);
if (magic.Magic != BitConverter.ToUInt32(LSFMagic.Signature, 0))
{
var msg = String.Format(
"Invalid LSF signature; expected {0,8:X}, got {1,8:X}",
BitConverter.ToUInt32(Header.Signature, 0), hdr.Magic
BitConverter.ToUInt32(LSFMagic.Signature, 0), magic.Magic
);
throw new InvalidDataException(msg);
}

if (hdr.Version < (ulong) LSFVersion.VerInitial || hdr.Version > (ulong) LSFVersion.MaxVersion)
if (magic.Version < (ulong) LSFVersion.VerInitial || magic.Version > (ulong) LSFVersion.MaxVersion)
{
var msg = String.Format("LSF version {0} is not supported", hdr.Version);
var msg = String.Format("LSF version {0} is not supported", magic.Version);
throw new InvalidDataException(msg);
}

this.Version = hdr.Version;
this.Version = (LSFVersion)magic.Version;

Metadata meta;
if (hdr.Version < (ulong)LSFVersion.VerBG3ExtendedHeader)
if (this.Version >= LSFVersion.VerBG3ExtendedHeader)
{
meta = BinUtils.ReadStruct<Metadata>(reader);
var hdr = BinUtils.ReadStruct<LSFHeaderV5>(reader);
GameVersion = PackedVersion.FromInt64(hdr.EngineVersion);
}
else
{
var meta2 = BinUtils.ReadStruct<MetadataV5>(reader);
meta = meta2.Metadata;
var hdr = BinUtils.ReadStruct<LSFHeader>(reader);
GameVersion = PackedVersion.FromInt32(hdr.EngineVersion);
}

var meta = BinUtils.ReadStruct<LSFMetadata>(reader);

Names = new List<List<String>>();
bool isCompressed = BinUtils.CompressionFlagsToMethod(meta.CompressionFlags) != CompressionMethod.None;
if (meta.StringsSizeOnDisk > 0 || meta.StringsUncompressedSize > 0)
Expand Down Expand Up @@ -332,11 +338,11 @@ public Resource Read()
}
}

Nodes = new List<NodeInfo>();
Nodes = new List<LSFNodeInfo>();
if (meta.NodesSizeOnDisk > 0 || meta.NodesUncompressedSize > 0)
{
uint onDiskSize = isCompressed ? meta.NodesSizeOnDisk : meta.NodesUncompressedSize;
var uncompressed = Decompress(reader, onDiskSize, meta.NodesUncompressedSize, hdr, meta);
var uncompressed = Decompress(reader, onDiskSize, meta.NodesUncompressedSize, meta);

#if DUMP_LSF_SERIALIZATION
using (var nodesFile = new FileStream("nodes.bin", FileMode.Create, FileAccess.Write))
Expand All @@ -347,17 +353,17 @@ public Resource Read()

using (var nodesStream = new MemoryStream(uncompressed))
{
var longNodes = hdr.Version >= (ulong)LSFVersion.VerExtendedNodes
var longNodes = Version >= LSFVersion.VerExtendedNodes
&& meta.HasSiblingData == 1;
ReadNodes(nodesStream, longNodes);
}
}

Attributes = new List<AttributeInfo>();
Attributes = new List<LSFAttributeInfo>();
if (meta.AttributesSizeOnDisk > 0 || meta.AttributesUncompressedSize > 0)
{
uint onDiskSize = isCompressed ? meta.AttributesSizeOnDisk : meta.AttributesUncompressedSize;
var uncompressed = Decompress(reader, onDiskSize, meta.AttributesUncompressedSize, hdr, meta);
var uncompressed = Decompress(reader, onDiskSize, meta.AttributesUncompressedSize, meta);

#if DUMP_LSF_SERIALIZATION
using (var attributesFile = new FileStream("attributes.bin", FileMode.Create, FileAccess.Write))
Expand All @@ -368,7 +374,7 @@ public Resource Read()

using (var attributesStream = new MemoryStream(uncompressed))
{
var hasSiblingData = hdr.Version >= (ulong)LSFVersion.VerExtendedNodes
var hasSiblingData = Version >= LSFVersion.VerExtendedNodes
&& meta.HasSiblingData == 1;
if (hasSiblingData)
{
Expand All @@ -384,7 +390,7 @@ public Resource Read()
if (meta.ValuesSizeOnDisk > 0 || meta.ValuesUncompressedSize > 0)
{
uint onDiskSize = isCompressed ? meta.ValuesSizeOnDisk : meta.ValuesUncompressedSize;
var uncompressed = Decompress(reader, onDiskSize, meta.ValuesUncompressedSize, hdr, meta);
var uncompressed = Decompress(reader, onDiskSize, meta.ValuesUncompressedSize, meta);
var valueStream = new MemoryStream(uncompressed);
this.Values = valueStream;

Expand All @@ -403,10 +409,10 @@ public Resource Read()
Resource resource = new Resource();
ReadRegions(resource);

resource.Metadata.MajorVersion = 1;
resource.Metadata.MinorVersion = 0;
resource.Metadata.Revision = 0;
resource.Metadata.BuildNumber = hdr.EngineVersion;
resource.Metadata.MajorVersion = GameVersion.Major;
resource.Metadata.MinorVersion = GameVersion.Minor;
resource.Metadata.Revision = GameVersion.Revision;
resource.Metadata.BuildNumber = GameVersion.Build;

return resource;
}
Expand Down Expand Up @@ -438,7 +444,7 @@ private void ReadRegions(Resource resource)
}
}

private void ReadNode(NodeInfo defn, Node node, BinaryReader attributeReader)
private void ReadNode(LSFNodeInfo defn, Node node, BinaryReader attributeReader)
{
node.Name = Names[defn.NameIndex][defn.NameOffset];

Expand Down Expand Up @@ -495,7 +501,7 @@ private NodeAttribute ReadAttribute(NodeAttribute.DataType type, BinaryReader re
var attr = new NodeAttribute(type);
var str = new TranslatedString();

if (Version >= (uint)LSFVersion.VerBG3)
if (Version >= LSFVersion.VerBG3)
{
str.Version = reader.ReadUInt16();
}
Expand Down Expand Up @@ -536,7 +542,7 @@ private TranslatedFSString ReadTranslatedFSString(BinaryReader reader)
{
var str = new TranslatedFSString();

if (Version >= (uint)LSFVersion.VerBG3)
if (Version >= LSFVersion.VerBG3)
{
str.Version = reader.ReadUInt16();
}
Expand Down
Loading

0 comments on commit 14a17db

Please sign in to comment.