lazy

//! The slot: LZ77 parsing, lazy matching over hash chains. A match found at `pos` is
//! held as pending while `pos + 1` is searched; a longer one there turns `input[pos]`
//! into a literal. Tokens: `t < 256` literal, else `2^24 + (dist-1)*256 + (len-3)`.

pub const MIN_MATCH: usize = 3;
pub const MAX_MATCH: usize = 258;
pub const WINDOW: usize = 32768;
pub const HASH_SIZE: usize = 32768;

/// How far down a hash chain to look. The main speed/ratio dial.
pub const MAX_PROBES: usize = 32;
/// Stop the chain walk once a match at least this long is found.
pub const NICE_LEN: usize = 128;
/// Do not look ahead when the pending match is already at least this long.
pub const LAZY_LIMIT: usize = 32;

/// Three bytes to a table index; `% 32768` so the bound is arithmetic. Correctness does not depend on it.
pub fn hash3(a: u8, b: u8, c: u8) -> usize {
    let x = (a as u32)
        .wrapping_mul(2654435761)
        .wrapping_add((b as u32).wrapping_mul(2246822519))
        .wrapping_add((c as u32).wrapping_mul(3266489917));
    ((x >> 15) % 32768) as usize
}

/// How many bytes agree at `a` and `b`, up to `cap`. The only function the proof depends on.
pub fn match_len(input: &[u8], a: usize, b: usize, cap: usize) -> usize {
    let mut l = 0usize;
    while l < cap && input[b + l] == input[a + l] {
        l += 1;
    }
    l
}

/// Walk a hash chain for the best `(length, distance)`; first found wins ties, so the nearest.
pub fn find_match(
    input: &[u8],
    prev: &[u32],
    pos: usize,
    cap: usize,
    start: usize,
) -> (usize, usize) {
    let mut best_len = 0usize;
    let mut best_dist = 0usize;
    let mut cur = start;
    let mut probes = 0usize;
    while probes < MAX_PROBES && cur > 0 && cur <= pos {
        let cpos = cur - 1;
        if pos - cpos <= 32768 {
            let l = match_len(input, cpos, pos, cap);
            if l > best_len {
                best_len = l;
                best_dist = pos - cpos;
            }
            if best_len >= NICE_LEN {
                cur = 0;
            } else {
                cur = prev[cpos % 32768] as usize;
            }
        } else {
            cur = 0;
        }
        probes += 1;
    }
    (best_len, best_dist)
}

/// Lazy matching: `pend_len >= 3` is a match found at `pos - 1`, not yet emitted because `pos` might beat it.
pub fn parse(input: &[u8], out: &mut [u32]) -> usize {
    let n = input.len();
    let mut head = [0u32; 32768];
    let mut prev = [0u32; 32768];
    let mut ntok = 0usize;
    let mut pos = 0usize;
    let has3 = n >= 3;
    let lim = if has3 { n - 3 } else { 0 };
    let mut pend_len = 0usize;
    let mut pend_dist = 0usize;
    while pos < n {
        let mut cur_len = 0usize;
        let mut cur_dist = 0usize;
        if has3 && pos <= lim {
            let h = hash3(input[pos], input[pos + 1], input[pos + 2]);
            let start = head[h] as usize;
            prev[pos % 32768] = head[h];
            head[h] = (pos + 1) as u32;
            if pend_len < LAZY_LIMIT {
                let mut cap = n - pos;
                if cap > 258 {
                    cap = 258;
                }
                let found = find_match(input, &prev, pos, cap, start);
                cur_len = found.0;
                cur_dist = found.1;
            }
        }
        if pend_len >= 3 {
            if pend_len >= cur_len {
                // The pending match wins: emit it, insert the positions it covers, continue after it.
                out[ntok] = 16777216u32 + ((pend_dist - 1) as u32) * 256 + ((pend_len - 3) as u32);
                ntok += 1;
                let end = pos - 1 + pend_len;
                let mut k = pos + 1;
                while k < end && has3 && k <= lim {
                    let h2 = hash3(input[k], input[k + 1], input[k + 2]);
                    prev[k % 32768] = head[h2];
                    head[h2] = (k + 1) as u32;
                    k += 1;
                }
                pos = end;
                pend_len = 0;
                pend_dist = 0;
            } else {
                // The match at `pos` is longer: `pos - 1` becomes a literal and `pos` is pending.
                out[ntok] = input[pos - 1] as u32;
                ntok += 1;
                pend_len = cur_len;
                pend_dist = cur_dist;
                pos += 1;
            }
        } else if cur_len >= 3 {
            pend_len = cur_len;
            pend_dist = cur_dist;
            pos += 1;
        } else {
            out[ntok] = input[pos] as u32;
            ntok += 1;
            pos += 1;
        }
    }
    // Provably unreachable (see Parse.lean); kept so no path leaves a token unemitted.
    if pend_len >= 3 {
        out[ntok] = 16777216u32 + ((pend_dist - 1) as u32) * 256 + ((pend_len - 3) as u32);
        ntok += 1;
    }
    ntok
}

import Lz77
import Slot

/-!
The lazy-matching proof. The search side is the hash-chains proof; the emission
side adds `Pending` (a match `Found` one position back) and `emitted` (the input
prefix the tokens so far account for).
-/

namespace Submission
open Aeneas Aeneas.Std Result ControlFlow

set_option maxRecDepth 8192
set_option maxHeartbeats 1000000

open LZ77 (toks bytes bytes_length bytes_getElem! toks_update bytes_congr
  Matches Found Pending emitted emitted_ge emitted_lt pending_of_found emit_lit emit_match)

/-! ## The hash: in range, and nothing else -/

@[local step]
theorem hash3_spec (a b c : Std.U8) :
    slot.hash3 a b c ⦃ fun h => h.val < 32768 ⦄ := by
  prove_hash3

/-! ## The match-length loop: the one load-bearing function -/

theorem match_len_loop_spec (input : Slice Std.U8) (a b cap l0 : Std.Usize)
    (ha : a.val + cap.val ≤ input.length) (hb : b.val + cap.val ≤ input.length)
    (hl0 : l0.val ≤ cap.val) (h0 : Matches input a.val b.val l0.val) :
    slot.match_len_loop input a b cap l0 ⦃ fun l =>
      l.val ≤ cap.val ∧ Matches input a.val b.val l.val ⦄ := by
  prove_match_len_loop

@[local step]
theorem match_len_spec (input : Slice Std.U8) (a b cap : Std.Usize)
    (ha : a.val + cap.val ≤ input.length) (hb : b.val + cap.val ≤ input.length) :
    slot.match_len input a b cap ⦃ fun l =>
      l.val ≤ cap.val ∧ Matches input a.val b.val l.val ⦄ := by
  prove_match_len

/-! ## The hash-insert loop: terminates, stays in bounds, proves nothing else -/

@[local step]
theorem parse_loop0_loop0_spec (input : Slice Std.U8)
    (head0 prev0 : Array Std.U32 32768#usize)
    (has30 : Bool) (lim «end» k0 : Std.Usize)
    (hlim : has30 = true → lim.val + 3 ≤ input.length) :
    slot.parse_loop0_loop0 input head0 prev0 has30 lim «end» k0
      ⦃ fun r => r.2.2 = true → lim.val + 3 ≤ input.length ⦄ := by
  rw [slot.parse_loop0_loop0]
  apply Std.loop.spec_decr_nat
    (measure := fun s => «end».val - s.2.2.2.val)
    (inv := fun s => s.2.2.1 = true → lim.val + 3 ≤ input.length)
  · rintro ⟨hd, pv, h3, k⟩ hinv
    have hmax : input.length ≤ Std.Usize.max := Std.Slice.length_ineq input
    simp only [slot.parse_loop0_loop0.body]
    step*
  · exact hlim

/-! ## The search: `Found` is the postcondition and the invariant; `cur` is never mentioned -/

theorem find_match_loop_spec (input : Slice Std.U8) (prev : Slice Std.U32)
    (n pos cap bl0 bd0 cur0 probes0 : Std.Usize)
    (hn : n.val = input.length) (hprev : prev.length = 32768)
    (hcap : pos.val + cap.val ≤ n.val) (hcap258 : cap.val ≤ 258)
    (h0 : Found input n pos bl0 bd0) :
    slot.find_match_loop input prev pos cap bl0 bd0 cur0 probes0
      ⦃ fun r => Found input n pos r.1 r.2 ⦄ := by
  rw [slot.find_match_loop]
  apply Std.loop.spec_decr_nat
    (measure := fun s => slot.MAX_PROBES.val - s.2.2.2.val)
    (inv := fun s => Found input n pos s.1 s.2.1)
  · rintro ⟨bl, bd, cur, probes⟩ hinv
    simp only at hinv
    have hmax : input.length ≤ Std.Usize.max := Std.Slice.length_ineq input
    simp only [slot.find_match_loop.body]
    step*
    apply Std.WP.spec_bind (Pₘ := fun r => Found input n pos r.1 r.2.1)
    · split
      case isTrue hw =>
        step*
        rw [show ((if l > bl then ok (l, i) else ok (bl, bd))
              : Result (Std.Usize × Std.Usize))
            = ok (if l > bl then (l, i) else (bl, bd)) from by split <;> rfl]
        step*
        split
        case isTrue hbetter =>
          apply Std.WP.spec_bind (Pₘ := fun (_ : Std.Usize) => True)
          · split <;> step*
          · intro cur1 _
            step*
            rcases Nat.lt_or_ge l.val 3 with h | h
            · exact Or.inl h
            · exact Or.inr ⟨h, by scalar_tac, by scalar_tac, by scalar_tac,
                by scalar_tac, by scalar_tac,
                by rw [show pos.val - i.val = cpos.val by scalar_tac]; exact l_post2⟩
        case isFalse =>
          apply Std.WP.spec_bind (Pₘ := fun (_ : Std.Usize) => True)
          · split <;> step*
          · intro cur1 _
            step*
      case isFalse => exact hinv
    · rintro ⟨bl1, bd1, cur1⟩ hf
      step*
  · exact h0

@[local step]
theorem find_match_spec (input : Slice Std.U8) (prev : Slice Std.U32)
    (n pos cap start : Std.Usize)
    (hn : n.val = input.length) (hprev : prev.length = 32768)
    (hcap : pos.val + cap.val ≤ n.val) (hcap258 : cap.val ≤ 258) :
    slot.find_match input prev pos cap start
      ⦃ fun r => Found input n pos r.1 r.2 ⦄ :=
  find_match_loop_spec input prev n pos cap 0#usize 0#usize start 0#usize
    hn hprev hcap hcap258 (Or.inl (by scalar_tac))

/-! ## The parse loop: tokens so far decode to `emitted`, and a pending match is `Pending` -/

theorem parse_loop0_spec (input : Slice Std.U8) (out0 : Slice Std.U32)
    (n lim : Std.Usize) (head0 prev0 : Array Std.U32 32768#usize)
    (ntok0 pos0 pl0 pd0 : Std.Usize) (has30 : Bool)
    (hn : n.val = input.length)
    (hout : input.length ≤ out0.length)
    (hlim : has30 = true → lim.val + 3 ≤ input.length)
    (hpos : pos0.val ≤ n.val) (hntok : ntok0.val ≤ emitted pos0.val pl0.val)
    (hpend : Pending input n pos0 pl0 pd0)
    (hdec : LZ77.decode (toks out0 ntok0.val) =
      some ((bytes input).take (emitted pos0.val pl0.val))) :
    slot.parse_loop0 input out0 n head0 prev0 ntok0 pos0 has30 lim pl0 pd0 ⦃ fun r =>
      r.2.2.1.val < 3 ∧ r.2.1.val ≤ input.length ∧ r.1.length = out0.length ∧
      LZ77.decode (toks r.1 r.2.1.val) = some (bytes input) ⦄ := by
  rw [slot.parse_loop0]
  apply Std.loop.spec_decr_nat
    (measure := fun s => n.val - s.2.2.2.2.1.val)
    (inv := fun s =>
      s.2.2.2.2.1.val ≤ n.val ∧
      s.2.2.2.1.val ≤ emitted s.2.2.2.2.1.val s.2.2.2.2.2.2.1.val ∧
      s.1.length = out0.length ∧
      (s.2.2.2.2.2.1 = true → lim.val + 3 ≤ input.length) ∧
      Pending input n s.2.2.2.2.1 s.2.2.2.2.2.2.1 s.2.2.2.2.2.2.2 ∧
      LZ77.decode (toks s.1 s.2.2.2.1.val) =
        some ((bytes input).take (emitted s.2.2.2.2.1.val s.2.2.2.2.2.2.1.val)))
  · rintro ⟨out, hd, pv, ntok, pos, h3, pl, pd⟩ ⟨hp, hnt, hlen, hh3, hpd, hde⟩
    simp only at hp hnt hlen hh3 hpd hde
    have hmax : input.length ≤ Std.Usize.max := Std.Slice.length_ineq input
    simp only [slot.parse_loop0.body]
    split
    case isTrue hposlt =>
      apply Std.WP.spec_bind (Pₘ := fun (r : (Array Std.U32 32768#usize) ×
          (Array Std.U32 32768#usize) × Std.Usize × Std.Usize) =>
        Found input n pos r.2.2.1 r.2.2.2)
      · split
        case isTrue hb3 =>
          split
          case isTrue hpl =>
            have hlim3 : lim.val + 3 ≤ input.length := hh3 hb3
            have hposl : pos.val ≤ lim.val := by scalar_tac
            step*
            apply Std.WP.spec_bind (Pₘ := fun (r : (Array Std.U32 32768#usize) ×
                Std.Usize × Std.Usize) => Found input n pos r.2.1 r.2.2)
            · split
              case isTrue hlazy =>
                step*
                rw [show ((if cap > 258#usize then ok 258#usize else ok cap)
                      : Result Std.Usize)
                    = ok (if cap > 258#usize then 258#usize else cap) from by
                      split <;> rfl]
                step*
                · split <;> scalar_tac
                · split <;> scalar_tac
              case isFalse => step*; exact Or.inl (by scalar_tac)
            · rintro ⟨a1, i8, i9⟩ hf
              step*
          case isFalse => exact Or.inl (by scalar_tac)
        case isFalse => exact Or.inl (by scalar_tac)
      · rintro ⟨head1, prev1, cur_len, cur_dist⟩ hfound
        replace hfound : Found input n pos cur_len cur_dist := hfound
        have hlim3 : h3 = true → lim.val + 3 ≤ input.length := hh3
        have hntok_lt : ntok.val < out.length := by
          have : emitted pos.val pl.val ≤ pos.val := by unfold emitted; split <;> omega
          scalar_tac
        -- `step*` splits every `if`; what remains is six range side-conditions of the
        -- pending emission, whose facts are in `Pending`, then one goal per branch.
        step*
        · rcases hpd with h | ⟨_, _, _, _, hd1, _⟩ <;> scalar_tac
        · rcases hpd with h | ⟨_, _, hlmax, _, hd1, hdmax, _⟩ <;> scalar_tac
        · rcases hpd with h | ⟨_, _, hlmax, _, hd1, hdmax, _⟩ <;> scalar_tac
        · rcases hpd with h | ⟨_, _, hlmax, _, hd1, hdmax, _⟩ <;> scalar_tac
        · rcases hpd with h | ⟨hpos1, _⟩ <;> scalar_tac
        · rcases hpd with h | ⟨hpos1, _, hlmax, hend, _⟩ <;> scalar_tac
        -- the pending match wins and is emitted at pos - 1
        · rcases hpd with h | ⟨hpos1, hl3, hlmax, hend, hd1, hdmax, hdpos, hmatch⟩
          · exfalso; scalar_tac
          rw [emitted_ge hl3] at hnt hde
          have htok : i6.val = LZ77.mkMatch pd.val pl.val := by
            simp only [LZ77.mkMatch, LZ77.MATCH_BASE, i6_post, i3_post, i2_post,
              i5_post, i1_post, i4_post1, i_post1, Std.UScalar.cast_val_eq]
            scalar_tac
          refine ⟨by scalar_tac, ?_, by rw [s_post]; simpa [Std.Slice.set_val_eq] using hlen,
            head2_post, Or.inl (by scalar_tac), ?_, by scalar_tac⟩
          · rw [emitted_lt (by scalar_tac)]; scalar_tac
          · rw [emitted_lt (by scalar_tac), s_post, show ntok1.val = ntok.val + 1 by scalar_tac,
              toks_update out ntok i6 hntok_lt, htok,
              show «end».val = (pos.val - 1) + pl.val by scalar_tac]
            refine LZ77.valid_match (bytes input) (toks out ntok.val) (pos.val - 1)
              pd.val pl.val hde hd1 hdpos
              (by simpa [LZ77.MAX_DIST] using hdmax) hl3
              (by simpa [LZ77.MAX_LEN] using hlmax) (by rw [bytes_length]; scalar_tac) ?_
            intro k hk
            exact (bytes_congr input _ _ (by scalar_tac) (by scalar_tac)
              (hmatch k hk)).symm
        · rcases hpd with h | ⟨hpos1, _⟩ <;> scalar_tac
        -- the match at pos is longer: pos - 1 becomes a literal, pos is pending
        · rcases hpd with h | ⟨hpos1, hl3, hlmax, hend, hd1, hdmax, hdpos, hmatch⟩
          · exfalso; scalar_tac
          rw [emitted_ge hl3] at hnt hde
          have hcur3 : 3 ≤ cur_len.val := by scalar_tac
          have hposlen : pos.val - 1 < input.length := by scalar_tac
          have hi : i.val = pos.val - 1 := by scalar_tac
          have hval : i2.val = (bytes input)[pos.val - 1]! := by
            rw [bytes_getElem! input (pos.val - 1) hposlen, i2_post, Std.U8.cast_U32_val_eq, i1_post]
            simp only [hi]
          refine ⟨by scalar_tac, ?_, by rw [s_post]; simpa [Std.Slice.set_val_eq] using hlen,
            hlim3, pending_of_found input n pos _ cur_len cur_dist (by scalar_tac) hfound,
            ?_, by scalar_tac⟩
          · rw [emitted_ge hcur3]; scalar_tac
          · rw [emitted_ge hcur3, s_post, show ntok1.val = ntok.val + 1 by scalar_tac,
              toks_update out ntok i2 hntok_lt, hval,
              show pos1.val - 1 = (pos.val - 1) + 1 by scalar_tac]
            exact LZ77.valid_lit (bytes input) (toks out ntok.val) (pos.val - 1) hde
              (by rw [bytes_length]; scalar_tac) (by rw [← hval]; scalar_tac)
        -- nothing pending and a match at pos: it becomes pending, nothing is written
        · rcases hpd with h | ⟨hpos1, hl3, _⟩
          · rw [emitted_lt h] at hnt hde
            have hcur3 : 3 ≤ cur_len.val := by scalar_tac
            refine ⟨by scalar_tac, ?_, hlen, hlim3,
              pending_of_found input n pos _ cur_len cur_dist (by scalar_tac) hfound,
              ?_, by scalar_tac⟩
            · rw [emitted_ge hcur3]; scalar_tac
            · rw [emitted_ge hcur3]; convert hde using 3; scalar_tac
          · exfalso; scalar_tac
        -- a plain literal
        · rcases hpd with h | ⟨hpos1, hl3, _⟩
          · rw [emitted_lt h] at hnt hde
            have hposlen : pos.val < input.length := by scalar_tac
            have hval : i1.val = (bytes input)[pos.val]! := by
              rw [bytes_getElem! input pos.val hposlen, i1_post,
                Std.U8.cast_U32_val_eq, i_post]
            refine ⟨by scalar_tac, ?_, by rw [s_post]; simpa [Std.Slice.set_val_eq] using hlen,
              hlim3, Or.inl h, ?_, by scalar_tac⟩
            · rw [emitted_lt h]; scalar_tac
            · rw [emitted_lt h, s_post, show ntok1.val = ntok.val + 1 by scalar_tac,
                toks_update out ntok i1 hntok_lt, hval,
                show pos1.val = pos.val + 1 by scalar_tac]
              exact LZ77.valid_lit (bytes input) (toks out ntok.val) pos.val hde
                (by rw [bytes_length]; scalar_tac) (by rw [← hval]; scalar_tac)
          · exfalso; scalar_tac
    case isFalse hge =>
      have hpn : pos.val = n.val := by scalar_tac
      -- a pending match would have to fit after n - 1: there is none
      have hpl : pl.val < 3 := by
        rcases hpd with h | ⟨_, hl3, _, hend, _⟩
        · exact h
        · exfalso; omega
      rw [emitted_lt hpl] at hnt hde
      refine ⟨hpl, by scalar_tac, hlen, ?_⟩
      rw [hde, hpn, hn]
      simp
  · exact ⟨hpos, hntok, rfl, hlim, hpend, hdec⟩

/-! ## The obligation -/

theorem parse_spec (input : Slice Std.U8) (out : Slice Std.U32)
    (hlen : input.length ≤ out.length) :
    slot.parse input out ⦃ fun r =>
      r.1.val ≤ input.length ∧
      r.2.length = out.length ∧
      LZ77.Valid (bytes input) (toks r.2 r.1.val) ⦄ := by
  rw [slot.parse]
  apply Std.WP.spec_bind (Pₘ := fun r => r.1 = true → r.2.val + 3 ≤ input.length)
  · split
    case isTrue h3 =>
      have hmax : input.length ≤ Std.Usize.max := Std.Slice.length_ineq input
      step*
    case isFalse h3 =>
      intro hc; exact absurd hc (by simp)
  · rintro ⟨has3, lim⟩ hlim
    apply Std.WP.spec_bind (parse_loop0_spec input out (Std.Slice.len input) lim
      (Std.Array.repeat 32768#usize 0#u32) (Std.Array.repeat 32768#usize 0#u32)
      0#usize 0#usize 0#usize 0#usize has3
      (by simp) hlen hlim (by scalar_tac) (by simp [emitted]) (Or.inl (by scalar_tac))
      (by simp [toks, LZ77.decode, emitted]))
    rintro ⟨out1, ntok, pl, pd⟩ ⟨hpl, hntok, hlen1, hdec⟩
    -- nothing is pending at the end, so `step*` closes the flush branch from `hpl`
    step*
    exact ⟨hntok, hlen1, hdec⟩

end Submission