Sanitizing packets

WowPacketParser/App.config

@@ -105,6 +105,8 @@
                           13: Binary (.pkt) merge output in a file (fusion.pkt)
                           14: UniversalProto (.dat)
                           15: UniversalProto with text (.dat)
+                          16: UniversalProto with separate text (.dat)
+                          17: SanitizedPkt (.pkt) removes personal data (e.g. guid, realmid, chat message, guild, name, ...) and dumps a new pkt (<name>_sanitized.pkt)
         -->
         <add key="DumpFormat" value="1"/>
 

WowPacketParser/Enums/DumpFormatType.cs

@@ -18,6 +18,7 @@ public enum DumpFormatType
         Fusion,
         UniversalProto,
         UniversalProtoWithText,
-        UniversalProtoWithSeparateText
+        UniversalProtoWithSeparateText,
+        SanitizedPkt
     }
 }

WowPacketParser/Loading/BinaryPacketReader.cs

@@ -30,11 +30,14 @@ enum PktVersion
         private int _snifferId;
         private short _snifferVersion;
 
+        private byte[] FileHeader;
+
         public BinaryPacketReader(SniffType type, string fileName, Encoding encoding)
         {
             _sniffType = type;
             _reader = new BinaryReader(new FileStream(@fileName, FileMode.Open, FileAccess.Read, FileShare.Read), encoding);
             ReadHeader();
+            StoreFileHeader();
         }
 
         void ReadHeader()
@@ -125,6 +128,16 @@ void ReadHeader()
             }
         }
 
+        void StoreFileHeader()
+        {
+            // all data till this position belong to the sniffFile header
+            // store the end position of the header
+            // then reset it to 0 and simply store the whole header again as byte[]
+            var currentPosition = _reader.BaseStream.Position;
+            _reader.BaseStream.Position = 0;
+            FileHeader = _reader.ReadBytes((int)currentPosition);
+        }
+
         static void SetBuild(uint build)
         {
             ClientVersion.SetVersion((ClientVersionBuild)build);
@@ -149,8 +162,13 @@ public Packet Read(int number, string fileName)
             byte[] data;
             StringBuilder writer = null;
             int cIndex = 0;
+            byte[] header;
             IPEndPoint endPoint = null; // Only used in PKT3.1 by TC's PacketLogger
 
+            // Note: PacketHeader ends before data
+            var packetHeaderStartPosition = _reader.BaseStream.Position;
+            long packetHeaderEndPosition = 0;
+
             if (_sniffType == SniffType.Pkt)
             {
                 switch (_pktVersion)
@@ -166,11 +184,13 @@ public Packet Read(int number, string fileName)
                         if (direction == Direction.ServerToClient)
                         {
                             opcode = _reader.ReadInt16();
+                            packetHeaderEndPosition = _reader.BaseStream.Position;
                             data = _reader.ReadBytes(length - 2);
                         }
                         else
                         {
                             opcode = _reader.ReadInt32();
+                            packetHeaderEndPosition = _reader.BaseStream.Position;
                             data = _reader.ReadBytes(length - 4);
                         }
 
@@ -256,6 +276,7 @@ public Packet Read(int number, string fileName)
                             _reader.ReadBytes(additionalSize);
 
                         opcode = _reader.ReadInt32();
+                        packetHeaderEndPosition = _reader.BaseStream.Position;
                         data = _reader.ReadBytes(length - 4);
                         break;
                     }
@@ -265,6 +286,7 @@ public Packet Read(int number, string fileName)
                         length = _reader.ReadInt32();
                         direction = (Direction)_reader.ReadByte();
                         time = Utilities.GetDateTimeFromUnixTime(_reader.ReadInt64());
+                        packetHeaderEndPosition = _reader.BaseStream.Position;
                         data = _reader.ReadBytes(length);
                         break;
                     }
@@ -276,6 +298,7 @@ public Packet Read(int number, string fileName)
                 length = _reader.ReadInt32();
                 time = Utilities.GetDateTimeFromUnixTime(_reader.ReadInt32());
                 direction = (Direction)_reader.ReadByte();
+                packetHeaderEndPosition = _reader.BaseStream.Position;
                 data = _reader.ReadBytes(length);
             }
 
@@ -286,6 +309,15 @@ public Packet Read(int number, string fileName)
                     ClientVersion.SetVersion(time);
             }
 
+            // store the end position of the packet so we can return here
+            var packetEndPosition = _reader.BaseStream.Position;
+
+            // go back to packetHeaderStartPosition
+            _reader.BaseStream.Position = packetHeaderStartPosition;
+            // store all bytes till packetHeaderEndPosition, because its the header
+            header = _reader.ReadBytes((int)(packetHeaderEndPosition - packetHeaderStartPosition));
+            // go back to end of packet
+            _reader.BaseStream.Position = packetEndPosition;
             // ignore opcodes that were not "decrypted" (usually because of
             // a missing session key) (only applicable to 335 or earlier)
             if (opcode >= 1312 && (ClientVersion.Build <= ClientVersionBuild.V3_3_5a_12340 && ClientVersion.Build != ClientVersionBuild.Zero))
@@ -294,7 +326,8 @@ public Packet Read(int number, string fileName)
             return new Packet(data, opcode, time, direction, number, writer, Path.GetFileName(fileName))
             {
                 ConnectionIndex = cIndex,
-                EndPoint = endPoint
+                EndPoint = endPoint,
+                Header = header
             };
         }
 
@@ -308,6 +341,11 @@ public long GetCurrentSize()
             return _reader?.BaseStream.Position ?? 0;
         }
 
+        public byte[] GetFileHeader()
+        {
+            return FileHeader;
+        }
+
         public void Dispose()
         {
             if (_reader == null) return;

WowPacketParser/Loading/IPacketReader.cs

@@ -9,5 +9,6 @@ public interface IPacketReader : IDisposable
         Packet Read(int number, string fileName);
         long GetTotalSize();
         long GetCurrentSize();
+        byte[] GetFileHeader();
     }
 }

WowPacketParser/Loading/SniffFile.cs

@@ -36,6 +36,8 @@ public class SniffFile
         private readonly List<string> _skippedHeaders = new List<string>();
         private readonly List<string> _noStructureHeaders = new List<string>();
 
+        private static byte _xorSeed;
+
         public SniffFile(string fileName, DumpFormatType dumpFormat = DumpFormatType.Text, Tuple<int, int> number = null)
         {
             if (string.IsNullOrWhiteSpace(fileName))
@@ -472,6 +474,137 @@ private void ProcessFileImpl()
                     FusionDump(packets);
                     break;
                 }
+                case DumpFormatType.SanitizedPkt:
+                {
+                    List<Packet> packetList = new List<Packet>();
+
+                    Random rnd = new Random();
+                    _xorSeed = (byte)rnd.Next(1, 255);
+
+                    var outFileName = Path.ChangeExtension(FileName, null) + "_sanitized.pkt";
+
+                    if (Utilities.FileIsInUse(outFileName))
+                    {
+                        Trace.WriteLine($"Save file {outFileName} is in use, parsing will not be done.");
+                        break;
+                    }
+                    File.Delete(outFileName);
+
+                    _stats.SetStartTime(DateTime.Now);
+
+                    var threadCount = Settings.Threads;
+                    if (threadCount == 0)
+                        threadCount = Environment.ProcessorCount;
+
+                    ThreadPool.SetMinThreads(threadCount + 2, 4);
+
+                    var written = false;
+                    var firstRead = true;
+
+                    var reader = _compression != FileCompression.None ? new Reader(_tempName, _sniffType) : new Reader(FileName, _sniffType);
+
+                    var pwp = new ParallelWorkProcessor<Packet>(() => // read
+                    {
+                        if (!reader.PacketReader.CanRead())
+                            return Tuple.Create<Packet, bool>(null, true);
+
+                        Packet packet;
+                        var b = reader.TryRead(out packet);
+
+                        if (firstRead)
+                        {
+                            Trace.WriteLine(
+                                $"{_logPrefix}: Parsing {Utilities.BytesToString(reader.PacketReader.GetTotalSize())} of packets. Detected version {ClientVersion.VersionString}");
+
+                            firstRead = false;
+                        }
+
+                        return Tuple.Create(packet, b);
+                    }, packet => // parse
+                    {
+                        // We have to parse the packets and then modify them
+                        if (packet.Direction == Direction.BNClientToServer ||
+                            packet.Direction == Direction.BNServerToClient)
+                            BattlenetHandler.ParseBattlenet(packet);
+                        else
+                            Handler.Parse(packet); // Attribute in handlers affect the Readers to write instead of read
+
+                        // Update statistics
+                        _stats.AddByStatus(packet.Status);
+                        return packet;
+                    },
+                    packet => // write
+                    {
+                        ShowPercentProgress("Processing...", reader.PacketReader.GetCurrentSize(), reader.PacketReader.GetTotalSize());
+
+                        if (!packet.Status.HasAnyFlag(Settings.OutputFlag) || !packet.WriteToFile)
+                        {
+                            packet.ClosePacket();
+                            return;
+                        }
+
+                        written = true;
+
+                        // get packet header if necessary
+                        if (Settings.LogPacketErrors)
+                        {
+                            switch (packet.Status)
+                            {
+                                case ParsedStatus.WithErrors:
+                                    _withErrorHeaders.Add(packet.GetHeader());
+                                    break;
+                                case ParsedStatus.NotParsed:
+                                    _skippedHeaders.Add(packet.GetHeader());
+                                    break;
+                                case ParsedStatus.NoStructure:
+                                    _noStructureHeaders.Add(packet.GetHeader());
+                                    break;
+                            }
+                        }
+
+                        // we can't sanitize stuff we can't read
+                        switch (packet.Status)
+                        {
+                            case ParsedStatus.WithErrors:
+                                _withErrorHeaders.Add(packet.GetHeader());
+                                packet.ClosePacket();
+                                return;
+                            case ParsedStatus.NotParsed:
+                                _skippedHeaders.Add(packet.GetHeader());
+                                packet.ClosePacket();
+                                return;
+                            case ParsedStatus.NoStructure:
+                                _noStructureHeaders.Add(packet.GetHeader());
+                                packet.ClosePacket();
+                                return;
+                        }
+
+                        // add packet to out list
+                        packetList.Add(packet);
+                    }, threadCount);
+
+                    pwp.WaitForFinished(Timeout.Infinite);
+
+                    reader.PacketReader.Dispose();
+
+                    _stats.SetEndTime(DateTime.Now);
+
+                    if (written)
+                        Trace.WriteLine($"{_logPrefix}: Saved file to '{outFileName}'");
+                    else
+                    {
+                        Trace.WriteLine($"{_logPrefix}: No file produced");
+                        File.Delete(outFileName);
+                    }
+
+                    Trace.WriteLine($"{_logPrefix}: {_stats}");
+
+                    SanitizedDump(outFileName, reader.PacketReader.GetFileHeader(), packetList);
+
+                    GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
+
+                    break;
+                }
                 default:
                 {
                     Trace.WriteLine($"{_logPrefix}: Dump format is none, nothing will be processed.");
@@ -560,6 +693,12 @@ private void BinaryDump(string fileName, ICollection<Packet> packets)
             BinaryPacketWriter.Write(SniffType.Pkt, fileName, Encoding.ASCII, packets);
         }
 
+        private void SanitizedDump(string fileName, byte[] fileHeader, ICollection<Packet> packets)
+        {
+            Trace.WriteLine($"{_logPrefix}: Copying {packets.Count} packets to .pkt format...");
+            SanitizedBinaryPacketWriter.Write(SniffType.Pkt, fileName, Encoding.ASCII, fileHeader, packets);
+        }
+
         private void WriteSQLs()
         {
             var sqlFileName = string.IsNullOrWhiteSpace(Settings.SQLFileName) ? $"{Utilities.FormattedDateTimeForFiles()}_{Path.GetFileName(FileName)}.sql" : Settings.SQLFileName;
@@ -658,5 +797,10 @@ private string GetCompressedFileName()
         {
             return FileName + _compression.GetExtension();
         }
+
+        public static byte GetXORSeed()
+        {
+            return _xorSeed;
+        }
     }
 }

WowPacketParser/Loading/SqLitePacketReader.cs

@@ -63,6 +63,11 @@ public long GetTotalSize()
             return _count;
         }
 
+        public byte[] GetFileHeader()
+        {
+            return new byte[0];
+        }
+
         public void Dispose()
         {
             if (_reader != null)

WowPacketParser/Misc/Packet.cs

@@ -33,6 +33,7 @@ public Packet(byte[] input, int opcode, DateTime time, Direction direction, int
             FileName = fileName;
             Status = ParsedStatus.None;
             WriteToFile = true;
+            BinaryWriter = new BinaryWriter(BaseStream);
 
             if (number == 0)
                 _firstPacketTime = Time;
@@ -54,6 +55,20 @@ public Packet(byte[] input, int opcode, DateTime time, Direction direction, int
         public Packet(byte[] input, int opcode, DateTime time, Direction direction, int number, string fileName)
             : this(input, opcode, time, direction, number, null, fileName)
         {
+            Opcode = opcode;
+            Time = time;
+            Direction = direction;
+            Number = number;
+            Writer = null;
+            FileName = fileName;
+            Status = ParsedStatus.None;
+            WriteToFile = true;
+            BinaryWriter = new BinaryWriter(BaseStream);
+
+            if (number == 0)
+                _firstPacketTime = Time;
+
+            TimeSpan = Time - _firstPacketTime;
         }
 
         public int Opcode { get; set; } // setter can't be private because it's used in multiple_packets
@@ -67,6 +82,8 @@ public Packet(byte[] input, int opcode, DateTime time, Direction direction, int
         public bool WriteToFile { get; private set; }
         public int ConnectionIndex { get; set; }
         public IPEndPoint EndPoint { get; set; }
+        public byte[] Header { get; set; }
+        public BinaryWriter BinaryWriter { get; }
 
         public PacketHolder Holder { get; set; }