Zend: Fix iterator relocation at the current element during rehash

NEWS

@@ -9,6 +9,8 @@ PHP                                                                        NEWS
 - Core:
   . Fixed out-of-bounds reads during automatic UTF-16/32 encoding detection.
     (Yudai Takada)
+  . Fixed incorrect internal pointer and foreach iterator positions when
+    compacting arrays with holes. (Weilin Du)
   . Fixed bug GH-15375 (Nested "yield from" skips items after a valid() or
     next() call on the inner generator). (iliaal)
   . Fixed bug GH-23232 (lone namespace separator asks the autoloader for an

Zend/tests/array_dup_internal_pointer_hole.phpt

@@ -0,0 +1,45 @@
+--TEST--
+Array duplication relocates an internal pointer on a hole, with and without foreach iterators
+--FILE--
+<?php
+function duplicate(array &$values): void {
+    $copy = $values;
+    $values['i'] = 18;
+    echo 'current: ', key($values), '=>', current($values), "\n";
+    next($values);
+    echo 'next: ', key($values), '=>', current($values), "\n";
+    echo 'copy current: ', key($copy), '=>', current($copy), "\n";
+    echo 'copy keys: ', implode(' ', array_keys($copy)), "\n";
+}
+
+foreach ([false, true] as $withIterator) {
+    echo $withIterator ? "With iterator:\n" : "Without iterator:\n";
+    $values = ['a' => 10, 'b' => 11, 'c' => 12, 'd' => 13,
+               'e' => 14, 'f' => 15, 'g' => 16, 'h' => 17];
+    next($values);
+    next($values);
+    // Leave the internal pointer on a hole before several surviving elements.
+    unset($values['a'], $values['b'], $values['c'], $values['d']);
+
+    if ($withIterator) {
+        foreach ($values as &$value) {
+            duplicate($values);
+            break;
+        }
+        unset($value);
+    } else {
+        duplicate($values);
+    }
+}
+?>
+--EXPECT--
+Without iterator:
+current: e=>14
+next: f=>15
+copy current: e=>14
+copy keys: e f g h
+With iterator:
+current: e=>14
+next: f=>15
+copy current: e=>14
+copy keys: e f g h

Zend/tests/array_dup_iterator_past_end.phpt

@@ -0,0 +1,23 @@
+--TEST--
+Array duplication preserves past-the-end iterators when compacting holes
+--FILE--
+<?php
+$values = ['a' => 10, 'b' => 11, 'c' => 12, 'd' => 13];
+unset($values['a'], $values['b']);
+
+foreach ($values as $key => &$value) {
+    echo "$key=>$value\n";
+    if ($key === 'd') {
+        // The iterator is one past the end; COW compacts the preceding holes.
+        $copy = $values;
+        $values['e'] = 14;
+    }
+}
+unset($value);
+echo 'copy: ', implode(' ', array_keys($copy)), "\n";
+?>
+--EXPECT--
+c=>12
+d=>13
+e=>14
+copy: c d

Zend/tests/array_dup_multiple_iterators.phpt

@@ -0,0 +1,38 @@
+--TEST--
+Array duplication updates iterators at both a hole and the next defined element
+--FILE--
+<?php
+function test(array $values): void {
+    $outerVisits = [];
+    $innerVisits = [];
+    $first = true;
+    foreach ($values as $outerKey => &$outerValue) {
+        $outerVisits[] = "$outerKey=>$outerValue";
+        if ($first) {
+            $first = false;
+            foreach ($values as $innerKey => &$innerValue) {
+                $innerVisits[] = "$innerKey=>$innerValue";
+                if ($innerValue === 11) {
+                    // The outer cursor is at a hole, the inner at the next value.
+                    unset($values['a'], $values['b']);
+                    $copy = $values;
+                    // Trigger copy-on-write duplication, which compacts the holes.
+                    $values['i'] = 18;
+                }
+            }
+            unset($innerValue);
+        }
+    }
+    unset($outerValue);
+    echo 'outer: ', implode(' ', $outerVisits), "\n";
+    echo 'inner: ', implode(' ', $innerVisits), "\n";
+    echo 'copy: ', implode(' ', array_keys($copy)), "\n";
+}
+
+test(['a' => 10, 'b' => 11, 'c' => 12, 'd' => 13,
+      'e' => 14, 'f' => 15, 'g' => 16, 'h' => 17]);
+?>
+--EXPECT--
+outer: a=>10 c=>12 d=>13 e=>14 f=>15 g=>16 h=>17 i=>18
+inner: a=>10 b=>11 c=>12 d=>13 e=>14 f=>15 g=>16 h=>17 i=>18
+copy: c d e f g h

Zend/tests/rehash_multiple_iterators.phpt

@@ -0,0 +1,40 @@
+--TEST--
+Rehashing updates iterators at both a hole and the next defined element
+--FILE--
+<?php
+function test(array $values, $firstKey, $secondKey, $newKey, int $newValue): void {
+    $outerVisits = [];
+    $innerVisits = [];
+    $first = true;
+    foreach ($values as $outerKey => &$outerValue) {
+        $outerVisits[] = "$outerKey=>$outerValue";
+        if ($first) {
+            $first = false;
+            foreach ($values as $innerKey => &$innerValue) {
+                $innerVisits[] = "$innerKey=>$innerValue";
+                if ($innerValue === 11) {
+                    // The outer cursor is at a hole, the inner at the next value.
+                    unset($values[$firstKey], $values[$secondKey]);
+                    $values[$newKey] = $newValue;
+                }
+            }
+            unset($innerValue);
+        }
+    }
+    unset($outerValue);
+    echo 'outer: ', implode(' ', $outerVisits), "\n";
+    echo 'inner: ', implode(' ', $innerVisits), "\n";
+}
+
+// Adding a string key converts packed storage and compacts its holes.
+test([10, 11, 12, 13, 14], 0, 1, 'new', 15);
+
+// Inserting into a full mixed table compacts its holes without growing it.
+test(['a' => 10, 'b' => 11, 'c' => 12, 'd' => 13,
+      'e' => 14, 'f' => 15, 'g' => 16, 'h' => 17], 'a', 'b', 'i', 18);
+?>
+--EXPECT--
+outer: 0=>10 2=>12 3=>13 4=>14 new=>15
+inner: 0=>10 1=>11 2=>12 3=>13 4=>14 new=>15
+outer: a=>10 c=>12 d=>13 e=>14 f=>15 g=>16 h=>17 i=>18
+inner: a=>10 b=>11 c=>12 d=>13 e=>14 f=>15 g=>16 h=>17 i=>18

Zend/zend_hash.c

@@ -1412,7 +1412,7 @@ ZEND_API void ZEND_FASTCALL zend_hash_rehash(HashTable *ht)
 								do {
 									zend_hash_iterators_update(ht, iter_pos, j);
 									iter_pos = zend_hash_iterators_lower_pos(ht, iter_pos + 1);
-								} while (iter_pos < i);
+								} while (iter_pos <= i);
 							}
 							q++;
 							j++;
@@ -2408,7 +2408,7 @@ static zend_always_inline uint32_t zend_array_dup_elements(HashTable *source, Ha
 			if (EXPECTED(!HT_HAS_ITERATORS(target))) {
 				while (p != end) {
 					if (zend_array_dup_element(source, target, target_idx, p, q, 0, static_keys, with_holes)) {
-						if (source->nInternalPointer == idx) {
+						if (UNEXPECTED(target->nInternalPointer > target_idx && target->nInternalPointer <= idx)) {
 							target->nInternalPointer = target_idx;
 						}
 						target_idx++; q++;
@@ -2421,19 +2421,21 @@ static zend_always_inline uint32_t zend_array_dup_elements(HashTable *source, Ha
 
 				while (p != end) {
 					if (zend_array_dup_element(source, target, target_idx, p, q, 0, static_keys, with_holes)) {
-						if (source->nInternalPointer == idx) {
+						if (UNEXPECTED(target->nInternalPointer > target_idx && target->nInternalPointer <= idx)) {
 							target->nInternalPointer = target_idx;
 						}
 						if (UNEXPECTED(idx >= iter_pos)) {
 							do {
 								zend_hash_iterators_update(target, iter_pos, target_idx);
 								iter_pos = zend_hash_iterators_lower_pos(target, iter_pos + 1);
-							} while (iter_pos < idx);
+							} while (iter_pos <= idx);
 						}
 						target_idx++; q++;
 					}
 					idx++; p++;
 				}
+				/* Move past-the-end iterators so they can pick up newly appended elements. */
+				_zend_hash_iterators_update(target, source->nNumUsed, target_idx);
 			}
 			return target_idx;
 		}