Deserializer: Array keys are not always 2 token positions apart.

Now keeping track of the end of arrays and objects and using that information when iterating over an array with either array or object values. Fixes #40

Author: StringEpsilon

CHANGELOG.md

@@ -1,4 +1,14 @@
-# Future
+# 2.0.1 (2024-11-18)
+
+## Bugfixes
+- Fixed validation error when deserializing a list of objects. The deserializer would check the wrong token for it's 
+  datatype and throw and exception like this:
+  `Can not deserialize array at position [x] to list: It has a non-integer key 'name' at element [y]` 
+  [GH #40](https://github.com/StringEpsilon/PhpSerializerNET/issues/40)
+- Related to the above: Some nested arrays or arrays with object values would never implicetly deserialize into a        
+  `List<object>` because the check if the array keys are consecutive integers was faulty.
+
+# 2.0.0 (2024-11-13)
 
 ## Breaking
 - Now targets .NET 8.0 and .NET 9.0

PhpSerializerNET.Test/Deserialize/ArrayDeserialization.cs

@@ -239,4 +239,25 @@ public void MixedKeyArrayIntoObject() {
 		Assert.Equal("A", result.Baz);
 		Assert.Equal("B", result.Dummy);
 	}
+
+	public class ArrayItem
+	{
+		[PhpProperty("foo")] public string Foo { get; set; }
+		[PhpProperty("bar")] public InnerArrayItem Bar { get; set; }
+	}
+
+	public class InnerArrayItem
+	{
+		public string A { get; set; }
+		public string B { get; set; }
+	}
+
+	[Fact]
+	public void NestedArrays() {
+		// Regression test for https://github.com/StringEpsilon/PhpSerializerNET/issues/40
+		var value = """a:2:{i:0;a:2:{s:3:"foo";s:4:"ixcg";s:3:"bar";a:2:{s:1:"A";s:5:"04381";s:1:"B";s:5:"11576";}}i:1;a:2:{s:3:"foo";s:4:"atnp";s:3:"bar";a:2:{s:1:"A";s:5:"33267";s:1:"B";s:5:"68391";}}}""";
+    	// PropertyType is
+		var item = PhpSerialization.Deserialize<List<ArrayItem>>(value);
+		Assert.NotNull(item);
+	}
 }

PhpSerializerNET.Test/Deserialize/ObjectDeserialization.cs

@@ -6,6 +6,7 @@ This Source Code Form is subject to the terms of the Mozilla Public
 **/
 
 using Xunit;
+using System.Collections.Generic;
 using PhpSerializerNET.Test.DataTypes;
 
 namespace PhpSerializerNET.Test.Deserialize;
@@ -23,4 +24,24 @@ public void IntegerKeysClass() {
 		Assert.Equal("A", result.Baz);
 		Assert.Equal("B", result.Dummy);
 	}
+
+	[Fact]
+	public void ListOfObjects() {
+		// Regression test for https://github.com/StringEpsilon/PhpSerializerNET/issues/40
+		var result = PhpSerialization.Deserialize<List<MixedKeysPhpClass>>(
+			"""a:3:{i:0;a:4:{s:3:"Foo";s:3:"Foo";s:3:"Bar";s:3:"Bar";s:1:"a";s:1:"A";s:1:"b";s:1:"B";}i:1;a:4:{s:3:"Foo";s:3:"Foo";s:3:"Bar";s:3:"Bar";s:1:"a";s:1:"A";s:1:"b";s:1:"B";}i:2;a:4:{s:3:"Foo";s:3:"Foo";s:3:"Bar";s:3:"Bar";s:1:"a";s:1:"A";s:1:"b";s:1:"B";}}"""
+		);
+
+		Assert.Equal(3, result.Count);
+	}
+	[Fact]
+	public void ImplicitListOfObjects() {
+		// Regression test for https://github.com/StringEpsilon/PhpSerializerNET/issues/40
+		var result = PhpSerialization.Deserialize(
+			"""a:3:{i:0;a:4:{s:3:"Foo";s:3:"Foo";s:3:"Bar";s:3:"Bar";s:1:"a";s:1:"A";s:1:"b";s:1:"B";}i:1;a:4:{s:3:"Foo";s:3:"Foo";s:3:"Bar";s:3:"Bar";s:1:"a";s:1:"A";s:1:"b";s:1:"B";}i:2;a:4:{s:3:"Foo";s:3:"Foo";s:3:"Bar";s:3:"Bar";s:1:"a";s:1:"A";s:1:"b";s:1:"B";}}""",
+			new PhpDeserializationOptions { UseLists = ListOptions.Default }
+		) as List<object>;
+
+		Assert.Equal(3, result.Count);
+	}
 }

PhpSerializerNET/Deserialization/PhpDeserializer.cs

@@ -395,14 +395,23 @@ private object MakeArray(Type targetType, in PhpToken token) {
 	}
 
 	private object MakeList(Type targetType, in PhpToken token) {
-		for (int i = 0; i < token.Length * 2; i += 2) {
-			if (this._tokens[this._currentToken + i].Type != PhpDataType.Integer) {
-				var badToken = this._tokens[this._currentToken + i];
+		int index = 0;
+		int itemPosition = this._currentToken;
+		while (index < token.Length) {
+			var valueToken = this._tokens[itemPosition +1];
+			if (this._tokens[itemPosition].Type != PhpDataType.Integer) {
+				var keyToken = this._tokens[itemPosition];
 				throw new DeserializationException(
 					$"Can not deserialize array at position {token.Position} to list: " +
-					$"It has a non-integer key '{this.GetString(badToken)}' at element {i} (position {badToken.Position})."
+					$"It has a non-integer key '{this.GetString(keyToken)}' at element {index+1} (position {keyToken.Position})."
 				);
 			}
+			index++;
+			if (valueToken.Type == PhpDataType.Array || valueToken.Type == PhpDataType.Object) {
+				itemPosition = valueToken.ValueEnd +1;
+			} else {
+				itemPosition += 2;
+			}
 		}
 
 		if (targetType.IsArray) {
@@ -468,18 +477,26 @@ private object MakeCollection(in PhpToken token) {
 		long previousKey = -1;
 		bool isList = true;
 		bool consecutive = true;
-		for (int i = 0; i < token.Length * 2; i += 2) {
-			if (this._tokens[this._currentToken + i].Type != PhpDataType.Integer) {
+		int index = 0;
+		int itemPosition = this._currentToken;
+		while (index < token.Length) {
+			if (this._tokens[itemPosition].Type != PhpDataType.Integer) {
 				isList = false;
 				break;
 			} else {
-				int key = this._tokens[this._currentToken + i].Value.GetInt(this._input);
-				if (i == 0 || key == previousKey + 1) {
+				int key = this._tokens[itemPosition].Value.GetInt(this._input);
+				if (index == 0 || key == previousKey + 1) {
 					previousKey = key;
 				} else {
 					consecutive = false;
 				}
 			}
+			index++;
+			if (this._tokens[itemPosition+1].Type == PhpDataType.Array || this._tokens[itemPosition+1].Type == PhpDataType.Object) {
+				itemPosition = this._tokens[itemPosition+1].ValueEnd +1;
+			} else {
+				itemPosition += 2;
+			}
 		}
 
 		if (!isList || (this._options.UseLists == ListOptions.Default && !consecutive)) {

PhpSerializerNET/Deserialization/PhpToken.cs

@@ -18,11 +18,27 @@ internal readonly struct PhpToken {
 	internal readonly int Position;
 	internal readonly int Length;
 	internal readonly ValueSpan Value;
+	/// <summary>
+	/// For <see cref="PhpDataType.Array"/> and <see cref="PhpDataType.Object"/> only. Holds the index of the last value
+	/// token inside the respective array/object.
+	/// </summary>
+	/// <remarks>
+	/// This does NOT reference the last value token. It could for example point to the last value token of an
+	/// object inside an array, when the "last value" of the array would be the object itself.
+	/// </remarks>
+	internal readonly int LastValuePosition;
 
-	internal PhpToken(in PhpDataType type, in int position, in ValueSpan value, int length = 0) {
+	internal PhpToken(
+		in PhpDataType type,
+		in int position,
+		in ValueSpan value,
+		int length = 0,
+		int lastValuePosition = 0
+	) {
 		this.Type = type;
 		this.Position = position;
 		this.Value = value;
 		this.Length = length;
+		this.LastValuePosition = lastValuePosition;
 	}
 }

PhpSerializerNET/Deserialization/PhpTokenizer.cs

@@ -121,41 +121,45 @@ private void GetFloatingToken() {
 
 	[MethodImpl(MethodImplOptions.AggressiveInlining)]
 	private void GetArrayToken() {
+		var tokenPosition = this._tokenPosition++;
 		int position = this._position - 1;
 		this._position++;
 		int length = this.GetLength();
-		this._tokens[this._tokenPosition++] = new PhpToken(
-			PhpDataType.Array,
-			position,
-			ValueSpan.Empty,
-			length
-		);
 		this.Advance(2);
 		for (int i = 0; i < length * 2; i++) {
 			this.GetToken();
 		}
+		this._tokens[tokenPosition] = new PhpToken(
+			PhpDataType.Array,
+			position,
+			ValueSpan.Empty,
+			length,
+			lastValuePosition: this._tokenPosition -1
+		);
 		this._position++;
 	}
 
 	[MethodImpl(MethodImplOptions.AggressiveInlining)]
 	private void GetObjectToken() {
+		var tokenPosition = this._tokenPosition++;
 		int position = this._position - 1;
 		this._position++;
 		int classNameLength = this.GetLength();
 		this.Advance(2);
 		ValueSpan classNameSpan = new ValueSpan(this._position, classNameLength);
 		this.Advance(2 + classNameLength);
 		int propertyCount = this.GetLength();
-		this._tokens[this._tokenPosition++] = new PhpToken(
-			PhpDataType.Object,
-			position,
-			classNameSpan,
-			propertyCount
-		);
 		this.Advance(2);
 		for (int i = 0; i < propertyCount * 2; i++) {
 			this.GetToken();
 		}
+		this._tokens[tokenPosition] = new PhpToken(
+			PhpDataType.Object,
+			position,
+			classNameSpan,
+			propertyCount,
+			lastValuePosition: this._tokenPosition -1
+		);
 		this._position++;
 	}