package puzzle;

import lombok.Getter;
import lombok.val;
import precomp.Neighbors9x8;
import precomp.Neighbors9x8.rci;
import puzzle.Export.Bit;
import puzzle.Export.Bit1029;
import puzzle.Export.DictEntryDTO;
import puzzle.Export.Gridded;
import puzzle.Export.Strings;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.stream.IntStream;
import static java.nio.charset.StandardCharsets.*;

/**
 * SwedishGenerator.java
 *
 * Usage:
 *   javac SwedishGenerator.java
 *   java SwedishGenerator [--seed N] [--pop N] [--gens N] [--tries N] [--words word-list.txt]
 */
@SuppressWarnings("ALL")
public record SwedishGenerator(Rng rng) {
   
   record CandidateInfo(int[] indices, int count) { }

//  static final CandidateInfo[] CANDIDATES = IntStream.range(0, 10192 << 2).mapToObj(CandidateInfo::new).toArray(CandidateInfo[]::new);
   
   //@formatter:off
   @FunctionalInterface interface SlotVisitor { void visit(int key, long lo, long hi); }
   //@formatter:on
   static final long   GT_1_OFFSET_53_BIT       = 0x3E00000000000000L;
   static final long   X                        = 0L;
   static final int    LOG_EVERY_MS             = 200;
   static final int    BAR_LEN                  = 22;
   static final int    C                        = Config.PUZZLE_COLS;
   static final double CROSS_R                  = (C - 1) / 2.0;
   static final int    R                        = Config.PUZZLE_ROWS;
   static final double CROSS_C                  = (R - 1) / 2.0;
   static final int    SIZE                     = C * R;// ~18
   static final int    SIZE_MIN_1               = SIZE - 1;// ~18
   static final double SIZED                    = (double) SIZE;// ~18
   static final int    TARGET_CLUES             = SIZE >>> 2;
   static final int    MAX_WORD_LENGTH          = C <= R ? C : R;
   static final int    MAX_WORD_LENGTH_PLUS_ONE = MAX_WORD_LENGTH + 1;
   static final int    MIN_LEN                  = Config.MIN_LEN;
   static final int    MIN_LEN7                 = Config.MIN_LEN * 7;
   static final int    MAX_TRIES_PER_SLOT       = Config.MAX_TRIES_PER_SLOT;
   static final char   C_DASH                   = '\0';
   static final byte   _1                       = 49, _9 = 57, A = 65, Z = 90, DASH = (byte) C_DASH;
   static final ThreadLocal<Context> CTX = ThreadLocal.withInitial(Context::new);
   
   static int clamp(int x, int a, int b) { return Math.max(a, Math.min(b, x)); }
   record Pick(Slot slot, CandidateInfo info, boolean done) { }
   
   static final byte    B0             = (byte) 0;
   static final byte    B64            = (byte) 64;
   static final byte    B48            = (byte) 48;
   static final long[]  OFFSETS_D_IDX  = Neighbors9x8.OFFSET_D_IDX;
   static final rci[]   IT             = Neighbors9x8.IT;
   static final int[][] MUTATE_RI      = new int[SIZE][625];
   static final long[]  NBR8_PACKED_LO = Neighbors9x8.NBR8_PACKED_LO;
   static final long[]  NBR8_PACKED_HI = Neighbors9x8.NBR8_PACKED_HI;
   static final long[]  PATH_LO        = Neighbors9x8.PATH_LO;
   static final long[]  PATH_HI        = Neighbors9x8.PATH_HI;
   
   static {
      for (int i = 0; i < SIZE; i++) {
         int k = 0;
         for (int dr1 = -2; dr1 <= 2; dr1++)
            for (int dr2 = -2; dr2 <= 2; dr2++)
               for (int dc1 = -2; dc1 <= 2; dc1++)
                  for (int dc2 = -2; dc2 <= 2; dc2++)
                       MUTATE_RI[i][k++] = Grid.offset(clamp(Grid.r(i) + dr1 + dr2, 0, R - 1),
                                                       clamp(Grid.c(i) + dc1 + dc2, 0, C - 1));
      }
   }
   
   static final Pick PICK_DONE     = new Pick(null, null, true);
   static final Pick PICK_NOT_DONE = new Pick(null, null, false);
   
   public static final class FillStats {
      
      public long   nodes;
      public long   backtracks;
      public double seconds;
      public int    lastMRV;
      public double simplicity;
   }
   
   public static record FillResult(boolean ok,
                                   Gridded grid,
                                   Map<Integer, Long> clueMap,
                                   FillStats stats) {
      
      public void calcSimpel() {
         if (ok) {
            clueMap.forEach((k, v) -> stats.simplicity += Lemma.simpel(v));
            stats.simplicity = clueMap.isEmpty() ? 0 : stats.simplicity / clueMap.size();
         }
      }
      
   }
   
   static final class Context {
      
      final Bit   covH2     = new Bit();
      final Bit   covV2     = new Bit();
      final int[] cellCount = new int[SIZE];
      final int[] stack     = new int[SIZE];
      final Bit   seen      = new Bit();
      long pattern;
      final long[] undo   = new long[128];
      final long[] bitset = new long[2500];
      
      void setPattern(long p) { this.pattern = p; }
   }
   
   static final class Rng {
      
      @Getter private int x;
      Rng(int seed) {
         var s = seed;
         if (s == 0) s = 1;
         this.x = s;
      }
      int nextU32() {
         var y = x;
         y ^= (y << 13);
         y ^= (y >>> 17);
         y ^= (y << 5);
         x = y;
         return y;
      }
      byte randbyte(int min, int max) {
         var u     = (nextU32() & 0xFFFFFFFFL);
         var range = (long) max - (long) min + 1L;
         return (byte) (min + (u % range));
      }
      int randint2bit() { return nextU32() & 3; }
      int randint(int min, int max) {
         var u     = (nextU32() & 0xFFFFFFFFL);
         var range = (long) max - (long) min + 1L;
         return (int) (min + (u % range));
      }
      double nextFloat() { return (nextU32() & 0xFFFFFFFFL) / 4294967295.0; }
   }
   
   static class Grid {
      
      final byte[] g;
      long lo, hi;
      
      public Grid(byte[] g) { this(g, 0, 0); }
      public Grid(byte[] g, long lo, long hi) {
         this.g  = g;
         this.lo = lo;
         this.hi = hi;
      }
      static Grid createEmpty()       { return new Grid(new byte[SIZE], X, X); }
      int digitAt(int index)          { return g[index] & 7; }
      public static int r(int offset) { return offset & 7; }
      public static int c(int offset) { return offset >>> 3; }
      static int offset(int r, int c) { return r | (c << 3); }
      Grid deepCopyGrid()             { return new Grid(g.clone(), lo, hi); }
      public byte byteAt(int pos)     { return g[pos]; }
      
      void setClue(int idx, byte ch) {
         g[idx] = ch;
         if ((idx & 64) == 0) lo |= (1L << idx);
         else hi |= (1L << (idx & 63));
      }
      void setLetter(int idx, byte ch) { g[idx] = ch; }
      void clearletter(int idx)        { g[idx] = DASH; }
      void clearClue(int idx) {
         g[idx] = DASH;
         if ((idx & 64) == 0) lo &= ~(1L << idx);
         else hi &= ~(1L << (idx & 63));
      }
      boolean isClue(long index)  { return ((index & 64) == 0) ? ((lo >>> index) & 1L) != X : ((hi >>> (index & 63)) & 1L) != X; }
      boolean isClue(int index)   { return ((index & 64) == 0) ? ((lo >>> index) & 1L) != X : ((hi >>> (index & 63)) & 1L) != X; }
      boolean notClue(long index) { return ((index & 64) == 0) ? ((lo >>> index) & 1L) == X : ((hi >>> (index & 63)) & 1L) == X; }
      boolean notClue(int index)  { return ((index & 64) == 0) ? ((lo >>> index) & 1L) == X : ((hi >>> (index & 63)) & 1L) == X; }
      boolean clueless(int idx) {
         if ((idx & 64) == 0) {
            val test = (1L << idx);
            if ((test & lo) == X) return false;
            g[idx] = DASH;
            
            lo &= ~test;
         } else {
            val test = (1L << (idx & 63));
            if ((test & hi) == X) return false;
            g[idx] = DASH;
            
            hi &= ~test;
         }
         return true;
      }
      
      public double similarity(Grid b) {
         var same = 0;
         for (int i = 0; i < SIZE; i++) if (g[i] == b.g[i]) same++;
         return same / SIZED;
      }
      int clueCount()                     { return Long.bitCount(lo) + Long.bitCount(hi); }
      boolean hasRoomForClue(long packed) { return (packed & GT_1_OFFSET_53_BIT) != X && notClue(packed & 0x7FL) && notClue((packed >>> 7) & 0x7FL); }
      void forEachSlot(SlotVisitor visitor) {
         for (var l = lo; l != X; l &= l - 1) processSlot(this, visitor, Long.numberOfTrailingZeros(l));
         for (var h = hi; h != X; h &= h - 1) processSlot(this, visitor, 64 | Long.numberOfTrailingZeros(h));
      }
      void undoPlace(long maskLo, long maskHi) {
         for (long b = maskLo; b != 0; b &= b - 1) clearletter(Long.numberOfTrailingZeros(b));
         for (long b = maskHi; b != 0; b &= b - 1) clearletter(64 | Long.numberOfTrailingZeros(b));
      }
   }
   
   static record DictEntry(long[] words, long[][] posBitsets) { }
   
   static int LEMMA_COUNTER = 0;
   
   public static interface Lemma {
      
      static final long LETTER_MASK = (1L << 40) - 1;     // low 40 bits
      static final long INDEX_MASK  = (1L << 24) - 1;     // 24 bits
      
      static long pack(String word)         { return pack(word.getBytes(US_ASCII)); }
      static long pack(int index, byte[] b) { return pack(b) | ((long) index << 40); }
      static long pack(byte[] b) {
         long w = 0;
         for (var i = 0; i < b.length; i++) w |= ((long) b[i] & ~64) << (i * 5);
         return w;
      }
      static public long from(String word)   { return pack(LEMMA_COUNTER++, word.getBytes(US_ASCII)); }
      static byte byteAt(long word, int idx) { return (byte) ((word >>> (idx * 5)) & 0b11111 | B64); }// word[]; }
      static int intAt(long word, int idx)   { return (int) (((word >>> (idx * 5))) & 0b11111); }// word[]; }
      static String[] clue(long w)           { return CsvIndexService.clues(unpackIndex(w)); }
      static int simpel(long w)              { return CsvIndexService.simpel(unpackIndex(w)); }
      static int length(long word)           { return ((63 - Long.numberOfLeadingZeros(word & LETTER_MASK)) / 5) + 1; }
      static int usedCharsInPattern(long p)  { return ((63 - Long.numberOfLeadingZeros(p & LETTER_MASK)) / 5) + 1; }
      public static String asWord(long word) {
         var b = new byte[Lemma.length(word)];
         for (var i = 0; i < b.length; i++) b[i] = (byte) ((word >>> (i * 5)) & 0b11111 | B64);
         return new String(b, US_ASCII);
      }
      static int unpackIndex(long w)   { return (int) (w >>> 40); }
      static int unpackLetters(long w) { return (int) (w & LETTER_MASK); }
   }
   
   public static record Dict(
           DictEntry[] index,
           int length) {
      
      public Dict(long[] wordz) {
         var index = new DictEntryDTO[MAX_WORD_LENGTH_PLUS_ONE];
         Arrays.setAll(index, i -> new DictEntryDTO(i));
         for (var lemma : wordz) {
            var L = Lemma.length(lemma);
            
            var entry = index[L];
            var idx   = entry.words().size();
            entry.words().add(lemma);
            
            for (var i = 0; i < L; i++) {
               var letter = Lemma.intAt(lemma, i) - 1;
               if (letter < 0 || letter >= 26) throw new RuntimeException("Illegal letter: " + letter + " in word " + lemma);
               entry.pos()[i][letter].add(idx);
            }
         }
         for (int i = MIN_LEN; i < index.length; i++) if (index[i].words().size() <= 0) throw new RuntimeException("No words for length " + i);
         this(Arrays.stream(index).map(i -> {
                 var words    = i.words().toArray();
                 int numWords = words.length;
                 int numLongs = (numWords + 63) >>> 6;
                 var bitsets  = new long[i.pos().length * 26][numLongs];
                 for (int p = 0; p < i.pos().length; p++) {
                    for (int l = 0; l < 26; l++) {
                       var list = i.pos()[p][l];
                       var bs   = bitsets[p * 26 + l];
                       for (int k = 0; k < list.size(); k++) {
                          int wordIdx = list.data()[k];
                          bs[wordIdx >>> 6] |= (1L << (wordIdx & 63));
                       }
                    }
                 }
                 return new DictEntry(words, bitsets);
              }).toArray(DictEntry[]::new),
              Arrays.stream(index).mapToInt(i -> i.words().size()).sum());
      }
      static Dict loadDict(String wordsPath) {
         try {
            var map = new LongArrayList(100_000);
            Files.lines(Path.of(wordsPath), UTF_8).forEach(line -> CsvIndexService.lineToLemma(line, map::add));
            return new Dict(map.toArray());
         } catch (IOException e) {
            e.printStackTrace();
            throw new RuntimeException("Failed to load dictionary from " + wordsPath, e);
         }
      }
   }
   
   static record Slot(int key, long lo, long hi) {
      
      static final int BIT_FOR_DIR = 3;
      static Slot from(int key, long lo, long hi)   { return new Slot(key, lo, hi); }
      
      public int len()                              { return Long.bitCount(lo) + Long.bitCount(hi); }
      public int clueR()                            { return Grid.r((key >>> BIT_FOR_DIR)); }
      public int clueIndex()                        { return clueIndex(key); }
      public static int clueIndex(int key)          { return key >>> BIT_FOR_DIR; }
      public int clueC()                            { return Grid.c((key >>> BIT_FOR_DIR)); }
      public int dir()                              { return key & 7; }
      public boolean horiz()                        { return horiz(key); }
      public boolean reversed()                     { return (key & 2) == 0; }
      public boolean increasing()                   { return (key & 2) != 0; }
      public static boolean increasing(int dir)     { return (dir & 2) != 0; }
      public IntStream walk()                       { return Gridded.walk((byte) key, lo, hi); }
      public static boolean horiz(int d)            { return (d & 1) == 0; }
      public static int packSlotDir(int idx, int d) { return (idx << BIT_FOR_DIR) | d; }
   }
   
   private static void processSlot(Grid grid, SlotVisitor visitor, int idx) {
      int d   = grid.digitAt(idx);         // 1..4
      int di  = d - 1;                    // 0..3
      int key = (idx << 2) | di;
      
      long rayLo = PATH_LO[key];
      long rayHi = PATH_HI[key];
      
      // only consider clue cells
      long hitsLo = rayLo & grid.lo;
      long hitsHi = rayHi & grid.hi;
      
      // slice ray to stop before first clue, depending on direction monotonicity
      // right/down => increasing indices; up/left => decreasing indices
      boolean increasing = Slot.increasing(d);
      
      if (increasing) {
         // first clue is lowest index among hits (lo first, then hi)
         if (hitsLo != 0) {
            long stop = 1L << Long.numberOfTrailingZeros(hitsLo);
            rayLo &= (stop - 1);
            rayHi = 0; // any hi is beyond the stop
         } else if (hitsHi != 0) {
            long stop = 1L << Long.numberOfTrailingZeros(hitsHi);
            // keep all lo (lo indices are < any hi index), but cut hi below stop
            rayHi &= (stop - 1);
         }
      } else {
         // first clue is highest index among hits (hi first, then lo)
         if (hitsHi != 0) {
            int  msb  = 63 - Long.numberOfLeadingZeros(hitsHi);
            long stop = 1L << msb;
            rayHi &= ~((stop << 1) - 1); // keep bits > stop
            rayLo = 0;                   // lo indices are below stop
         } else if (hitsLo != 0) {
            int  msb  = 63 - Long.numberOfLeadingZeros(hitsLo);
            long stop = 1L << msb;
            rayLo &= ~((stop << 1) - 1);
         }
      }
      
      if ((rayLo | rayHi) != 0) {
         visitor.visit(Slot.packSlotDir(idx, d), rayLo, rayHi);
      }
   }
   
   static ArrayList<Slot> extractSlots(Grid grid) {
      var slots = new ArrayList<Slot>(32);
      grid.forEachSlot((key, lo, hi) -> slots.add(Slot.from(key, lo, hi)));
      return slots;
   }
   
   long maskFitness(Grid grid) {
      var ctx  = CTX.get();
      var covH = ctx.covH2;
      var covV = ctx.covV2;
      covH.clear();
      covV.clear();
      long    lo_cl    = grid.lo, hi_cl = grid.hi;
      long    penalty  = (((long) Math.abs(grid.clueCount() - TARGET_CLUES)) * 16000L);
      boolean hasSlots = false;
      
      for (int i = 0; i < 65; i += 64) {
         for (long bits = (i == 0 ? lo_cl : hi_cl); bits != X; bits &= bits - 1) {
            int  clueIdx = i | Long.numberOfTrailingZeros(bits);
            int  d       = grid.digitAt(clueIdx);
            int  di      = d - 1;
            int  key     = (clueIdx << 2) | di;
            long rLo     = PATH_LO[key], rHi = PATH_HI[key];
            long hLo     = rLo & lo_cl, hHi = rHi & hi_cl;
            if (Slot.increasing(d)) {
               if (hLo != 0) {
                  rLo &= ((1L << Long.numberOfTrailingZeros(hLo)) - 1);
                  rHi = 0;
               } else if (hHi != 0) { rHi &= ((1L << Long.numberOfTrailingZeros(hHi)) - 1); }
            } else {
               if (hHi != 0) {
                  int msb = 63 - Long.numberOfLeadingZeros(hHi);
                  rHi &= ~((1L << msb << 1) - 1);
                  rLo = 0;
               } else if (hLo != 0) {
                  int msb = 63 - Long.numberOfLeadingZeros(hLo);
                  rLo &= ~((1L << msb << 1) - 1);
               }
            }
            if ((rLo | rHi) != 0) {
               hasSlots = true;
               if (Slot.horiz(d)) covH.or(rLo, rHi);
               else covV.or(rLo, rHi);
               if ((Long.bitCount(rLo) + Long.bitCount(rHi)) < MIN_LEN) penalty += 8000;
            } else {
               penalty += 25000;
            }
         }
      }
      
      if (!hasSlots) return 1_000_000_000L;
      
      var  stack  = ctx.stack;
      long seenLo = 0L, seenHi = 0L;

// loop over beide helften
      for (int base = 0; base <= 64; base += 64) {
         long clueMask = (base == 0) ? lo_cl : hi_cl;
         long seenMask = (base == 0) ? seenLo : seenHi;
         
         // "unseen clues" in deze helft
         for (long bits = clueMask & ~seenMask; bits != 0L; bits &= bits - 1) {
            int clueIdx = base + Long.numberOfTrailingZeros(bits);
            
            // start nieuwe component
            int size = 0;
            int sp   = 0;
            stack[sp++] = clueIdx;
            
            // mark seen
            if ((clueIdx & 64) == 0) seenLo |= 1L << clueIdx;
            else seenHi |= 1L << (clueIdx & 63);
            
            // flood fill / bfs
            while (sp > 0) {
               int cur = stack[--sp];
               size++;
               
               // neighbors als 2x long masks
               long nLo = NBR8_PACKED_LO[cur];
               long nHi = NBR8_PACKED_HI[cur];
               
               // filter: alleen clues, en nog niet seen
               nLo &= lo_cl & ~seenLo;
               nHi &= hi_cl & ~seenHi;
               
               // push lo-neighbors
               while (nLo != 0L) {
                  long lsb  = nLo & -nLo;
                  int  nidx = Long.numberOfTrailingZeros(nLo); // 0..63
                  seenLo |= lsb;
                  
                  stack[sp++] = nidx;
                  
                  nLo &= nLo - 1;
               }
               
               // push hi-neighbors
               while (nHi != 0L) {
                  long lsb  = nHi & -nHi;
                  int  nidx = 64 | Long.numberOfTrailingZeros(nHi); // 64..127
                  seenHi |= lsb;
                  
                  stack[sp++] = nidx;
                  
                  nHi &= nHi - 1;
               }
            }
            
            if (size >= 2) penalty += (size - 1L) * 120L;
         }
      }
      
      for (int i = 0; i < 65; i += 64) {
         long bits = (i == 0 ? ~lo_cl : (~hi_cl & 0xFFL));
         for (; bits != X; bits &= bits - 1) {
            int clueIdx = i + Long.numberOfTrailingZeros(bits);
            var rci     = IT[clueIdx];
            if ((4 - rci.nbrCount()) + Long.bitCount(rci.n1() & lo_cl) + Long.bitCount(rci.n2() & hi_cl) >= 3) penalty += 400;
            var h = covH.get(clueIdx);
            var v = covV.get(clueIdx);
            if (!h && !v) penalty += 1500;
            else if (h && v) { /* ok */ } else if (h | v) penalty += 200;
            else penalty += 600;
         }
      }
      
      return penalty;
   }
   
   Grid randomMask() {
      var g = Grid.createEmpty();
      for (int placed = 0, guard = 0, idx; placed < TARGET_CLUES && guard < 4000; guard++) {
         idx = rng.randint(0, SIZE_MIN_1);
         if (g.isClue(idx)) continue;
         int d_idx = rng.randint2bit();
         if (g.hasRoomForClue(OFFSETS_D_IDX[d_idx | idx << 2])) {
            g.setClue(idx, (byte) (1 + (d_idx | 48)));
            placed++;
         }
      }
      return g;
   }
   Grid mutate(Grid grid) {
      var g     = grid.deepCopyGrid();
      int ri;
      var bytes = MUTATE_RI[rng.randint(0, SIZE_MIN_1)];
      for (var k = 0; k < 4; k++) {
         ri = bytes[rng.randint(0, 624)];
         if (!g.clueless(ri)) {
            int d_idx  = rng.randint2bit();
            val packed = OFFSETS_D_IDX[d_idx | ri << 2];
            if (g.hasRoomForClue(packed)) g.setClue(ri, (byte) (1 + (d_idx | 48)));
         }
      }
      return g;
   }
   Grid crossover(Grid a, Grid b) {
      var out   = a.deepCopyGrid();
      var theta = rng.nextFloat() * Math.PI;
      var nc    = Math.cos(theta);
      var nr    = Math.sin(theta);
      
      long bo0 = out.lo, bo1 = out.hi;
      for (var rci : IT) {
         int i = rci.i();
         if ((rci.cross_r()) * nc + (rci.cross_c()) * nr < 0) {
            byte ch = b.g[i];
            if (out.g[i] != ch) {
               out.g[i] = ch;
               if (b.isClue(i)) {
                  if ((i & 64) == 0) bo0 |= (1L << i);
                  else bo1 |= (1L << (i & 63));
               } else {
                  if ((i & 64) == 0) bo0 &= ~(1L << i);
                  else bo1 &= ~(1L << (i & 63));
               }
            }
         }
      }
      out.lo = bo0;
      out.hi = bo1;
      for (var lo = out.lo; lo != X; lo &= lo - 1L) clearClues(out, Long.numberOfTrailingZeros(lo));
      for (var hi = out.hi; hi != X; hi &= hi - 1L) clearClues(out, 64 + Long.numberOfTrailingZeros(hi));
      return out;
   }
   public static void clearClues(Grid out, int idx) { if (!out.hasRoomForClue(OFFSETS_D_IDX[(out.digitAt(idx) - 1) | (idx << 2)])) out.clearClue(idx); }
   
   Grid hillclimb(Grid start, int limit) {
      var best  = start;
      var bestF = maskFitness(best);
      var fails = 0;
      
      while (fails < limit) {
         var cand = mutate(best);
         var f    = maskFitness(cand);
         if (f < bestF) {
            best  = cand;
            bestF = f;
            fails = 0;
         } else {
            fails++;
         }
      }
      return best;
   }
   
   public Grid generateMask(int popSize, int gens, int pairs) {
      class GridAndFit {
         
         Grid grid;
         long fite = -1;
         GridAndFit(Grid grid) { this.grid = grid; }
         long fit() {
            if (fite == -1) this.fite = maskFitness(grid);
            return this.fite;
         }
      }
      if (Main.VERBOSE) System.out.println("generateMask init pop: " + popSize);
      var pop = new ArrayList<GridAndFit>();
      for (var i = 0; i < popSize; i++) {
         pop.add(new GridAndFit(hillclimb(randomMask(), 180)));
      }
      
      for (var gen = 0; gen < gens; gen++) {
         if (Thread.currentThread().isInterrupted()) break;
         var children = new ArrayList<GridAndFit>();
         
         for (var k = 0; k < pairs; k++) {
            var p1    = pop.get(rng.randint(0, pop.size() - 1));
            var p2    = pop.get(rng.randint(0, pop.size() - 1));
            var child = crossover(p1.grid, p2.grid);
            children.add(new GridAndFit(hillclimb(child, 70)));
         }
         
         pop.addAll(children);
         pop.sort(Comparator.comparingLong(GridAndFit::fit));
         
         var next = new ArrayList<GridAndFit>();
         for (var cand : pop) {
            if (next.size() >= popSize) break;
            var ok = true;
            for (var kept : next) {
               if (cand.grid.similarity(kept.grid) > 0.92) {
                  ok = false;
                  break;
               }
            }
            if (ok) next.add(cand);
         }
         pop = next;
         
         if (Main.VERBOSE && (gen & 15) == 15) System.out.println("  gen " + gen + "/" + gens + " bestFitness=" + pop.get(0).fit());
      }
      GridAndFit best = pop.get(0);
      for (int i = 1; i < pop.size(); i++) {
         var x = pop.get(i);
         if (x.fit() < best.fit()) best = x;
      }
      return best.grid;
   }
   
   static long patternForSlot(Grid grid, Slot s) {
      if ((s.lo & ~grid.lo) == 0 && (s.hi & ~grid.hi) == 0) return 0;
      long p = 0;
      if (s.increasing()) {
         for (long b = s.lo & ~grid.lo; b != 0; b &= b - 1) {
            int idx = Long.numberOfTrailingZeros(b);
            int i   = Long.bitCount(s.lo & ((1L << idx) - 1));
            p |= ((long) (grid.g[idx] & 31)) << (i * 5);
         }
         int offset = Long.bitCount(s.lo);
         for (long b = s.hi & ~grid.hi; b != 0; b &= b - 1) {
            int idx = Long.numberOfTrailingZeros(b);
            int i   = offset + Long.bitCount(s.hi & ((1L << idx) - 1));
            p |= ((long) (grid.g[64 | idx] & 31)) << (i * 5);
         }
      } else {
         int offset = Long.bitCount(s.hi);
         for (long b = s.hi & ~grid.hi; b != 0; b &= b - 1) {
            int idx = Long.numberOfTrailingZeros(b);
            int i   = Long.bitCount(s.hi & ~((1L << idx) | ((1L << idx) - 1)));
            p |= ((long) (grid.g[64 | idx] & 31)) << (i * 5);
         }
         for (long b = s.lo & ~grid.lo; b != 0; b &= b - 1) {
            int idx = Long.numberOfTrailingZeros(b);
            int i   = offset + Long.bitCount(s.lo & ~((1L << idx) | ((1L << idx) - 1)));
            p |= ((long) (grid.g[idx] & 31)) << (i * 5);
         }
      }
      return p;
   }
   static int slotScore(int[] count, Slot s) {
      int cross = 0;
      for (long b = s.lo; b != 0; b &= b - 1) cross += (count[Long.numberOfTrailingZeros(b)] - 1);
      for (long b = s.hi; b != 0; b &= b - 1) cross += (count[64 | Long.numberOfTrailingZeros(b)] - 1);
      return cross * 10 + s.len();
   }
   static boolean placeWord(Grid grid, Slot s, long w, long[] undoBuffer, int offset) {
      long maskLo = 0, maskHi = 0;
      int  i      = 0;
      if (s.increasing()) {
         for (long b = s.lo; b != 0; b &= b - 1, i++) {
            int  idx = Long.numberOfTrailingZeros(b);
            byte cur = grid.byteAt(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskLo |= (1L << idx);
               grid.setLetter(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
         }
         for (long b = s.hi; b != 0; b &= b - 1, i++) {
            int  idx = 64 | Long.numberOfTrailingZeros(b);
            byte cur = grid.byteAt(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskHi |= (1L << (idx & 63));
               grid.setLetter(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
         }
      } else {
         for (long b = s.hi; b != 0; i++) {
            int  msb = 63 - Long.numberOfLeadingZeros(b);
            int  idx = 64 | msb;
            byte cur = grid.byteAt(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskHi |= (1L << msb);
               grid.setLetter(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
            b &= ~(1L << msb);
         }
         for (long b = s.lo; b != 0; i++) {
            int  msb = 63 - Long.numberOfLeadingZeros(b);
            int  idx = msb;
            byte cur = grid.byteAt(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskLo |= (1L << msb);
               grid.setLetter(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
            b &= ~(1L << msb);
         }
      }
      undoBuffer[offset << 1]       = maskLo;
      undoBuffer[(offset << 1) | 1] = maskHi;
      return true;
   }
   
   static CandidateInfo candidateInfoForPattern(Context ctx, DictEntry entry, int lenb) {
      var pattern = ctx.pattern;
      if (pattern == X) {
         return new CandidateInfo(null, entry.words.length);
      }
      
      int     numLongs = (entry.words.length + 63) >>> 6;
      long[]  res      = ctx.bitset;
      boolean first    = true;
      
      for (int i = 0, len = Lemma.usedCharsInPattern(pattern); i < len; i++) {
         int val = (int) ((pattern >>> (i * 5)) & 31);
         if (val != 0) {
            long[] bs = entry.posBitsets[i * 26 + (val - 1)];
            if (first) {
               System.arraycopy(bs, 0, res, 0, numLongs);
               first = false;
            } else {
               for (int k = 0; k < numLongs; k++) res[k] &= bs[k];
            }
         }
      }
      
      int count = 0;
      for (int k = 0; k < numLongs; k++) count += Long.bitCount(res[k]);
      
      if (count == 0) return new CandidateInfo(null, 0);
      
      int[] indices = new int[count];
      int   ki      = 0;
      for (int k = 0; k < numLongs; k++) {
         long w = res[k];
         while (w != 0) {
            int t = Long.numberOfTrailingZeros(w);
            indices[ki++] = (k << 6) | t;
            w &= w - 1;
         }
      }
      
      return new CandidateInfo(indices, count);
   }
   
   static int candidateCountForPattern(Context ctx, DictEntry entry) {
      long pattern = ctx.pattern;
      if (pattern == X) return entry.words.length;
      
      int     numLongs = (entry.words.length + 63) >>> 6;
      long[]  res      = ctx.bitset;
      boolean first    = true;
      
      for (int i = 0, len = Lemma.usedCharsInPattern(pattern); i < len; i++) {
         int val = (int) ((pattern >>> (i * 5)) & 31);
         if (val != 0) {
            long[] bs = entry.posBitsets[i * 26 + (val - 1)];
            if (first) {
               System.arraycopy(bs, 0, res, 0, numLongs);
               first = false;
            } else {
               for (int k = 0; k < numLongs; k++) res[k] &= bs[k];
            }
         }
      }
      
      if (first) return entry.words.length; // should not happen if pattern != X
      
      int count = 0;
      for (int k = 0; k < numLongs; k++) count += Long.bitCount(res[k]);
      return count;
   }
   //72 << 3;
   static final int BIGG = 581 + 1;
   public FillResult fillMask(Grid mask, DictEntry[] dictIndex,
                              int timeLimitMs) {
      val multiThreaded = Thread.currentThread().getName().contains("pool");
      val grid          = mask.deepCopyGrid();
      val used          = new Bit1029();
      
      long[] assigned = new long[BIGG];
      val    ctx      = CTX.get();
      val    count    = ctx.cellCount;
      Arrays.fill(count, 0, SIZE, 0);
      
      val slots = extractSlots(grid);
      for (var s : slots) {
         for (long b = s.lo; b != 0; b &= b - 1) count[Long.numberOfTrailingZeros(b)]++;
         for (long b = s.hi; b != 0; b &= b - 1) count[64 | Long.numberOfTrailingZeros(b)]++;
      }
      
      val slotScores = new int[slots.size()];
      for (int i = 0; i < slots.size(); i++) slotScores[i] = slotScore(count, slots.get(i));
      
      val t0    = System.currentTimeMillis();
      val stats = new FillStats();
      val TOTAL = slots.size();
      
      class Solver {
         
         long lastLog = t0;
         void renderProgress() {
            if (!Main.VERBOSE || multiThreaded) return;
            var now = System.currentTimeMillis();
            if ((now - lastLog) < LOG_EVERY_MS) return;
            lastLog = (now);
            var done = 0;
            for (var lemma : assigned) {
               if (lemma != X) done++;
            }
            var pct     = (TOTAL == 0) ? 100 : (int) Math.floor((done / (double) TOTAL) * 100);
            var filled  = Math.min(BAR_LEN, (int) Math.floor((pct / 100.0) * BAR_LEN));
            var bar     = "[" + "#".repeat(filled) + "-".repeat(BAR_LEN - filled) + "]";
            var elapsed = String.format(Locale.ROOT, "%.1fs", (now - t0) / 1000.0);
            
            var msg = String.format(
                    Locale.ROOT,
                    "%s %d/%d slots | nodes=%d | backtracks=%d | mrv=%d | %s",
                    bar, done, TOTAL, stats.nodes, stats.backtracks, stats.lastMRV, elapsed
                                   );
            System.out.print("\r" + Strings.padRight(msg, 120));
            System.out.flush();
         }
         Pick chooseMRV() {
            Slot          best      = null;
            CandidateInfo bestInfo  = null;
            int           bestScore = -1;
            for (int i = 0, n = slots.size(); i < n; i++) {
               var s = slots.get(i);
               if (assigned[s.key()] != X) continue;
               var entry = dictIndex[s.len()];
               if (entry == null) return PICK_NOT_DONE;
               ctx.pattern = patternForSlot(grid, s);
               int count = candidateCountForPattern(ctx, entry);
               
               if (count == 0) return PICK_NOT_DONE;
               if (best == null
                   || count < bestInfo.count
                   || (count == bestInfo.count && slotScores[i] > bestScore)) {
                  best      = s;
                  bestScore = slotScores[i];
                  bestInfo  = new CandidateInfo(null, count);
                  if (count <= 1) break;
               }
            }
            
            if (best == null) return PICK_DONE;
            
            // Re-calculate for the best slot to get actual indices
            ctx.pattern = patternForSlot(grid, best);
            bestInfo    = candidateInfoForPattern(ctx, dictIndex[best.len()], best.len());
            
            return new Pick(best, bestInfo, false);
         }
         boolean backtrack(int depth) {
            if (Thread.currentThread().isInterrupted()) return false;
            stats.nodes++;
            
            if (timeLimitMs > 0 && (System.currentTimeMillis() - t0) > timeLimitMs) return false;
            
            var pick = chooseMRV();
            if (pick.done) return true;
            if (pick.slot == null) {
               stats.backtracks++;
               return false;
            }
            val info = pick.info;
            stats.lastMRV = info.count;
            renderProgress();
            
            var s      = pick.slot;
            var k      = s.key();
            int patLen = s.len();
            var entry  = dictIndex[patLen];
            
            if (info.indices != null && info.indices.length > 0) {
               var idxs  = info.indices;
               var L     = idxs.length;
               var tries = Math.min(MAX_TRIES_PER_SLOT, L);
               
               for (var t = 0; t < tries; t++) {
                  double r          = rng.nextFloat();
                  int    idxInArray = (int) (r * r * r * L);
                  var    idx        = idxs[idxInArray];
                  var    w          = entry.words[idx];
                  var    lemIdx     = Lemma.unpackIndex(w);
                  if (used.get(lemIdx)) continue;
                  
                  if (!placeWord(grid, s, w, ctx.undo, depth)) continue;
                  
                  used.set(lemIdx);
                  assigned[k] = w;
                  
                  if (backtrack(depth + 1)) return true;
                  
                  assigned[k] = X;
                  used.clear(lemIdx);
                  grid.undoPlace(ctx.undo[depth << 1], ctx.undo[(depth << 1) | 1]);
               }
               stats.backtracks++;
               return false;
            }
            
            var N = entry.words.length;
            if (N == 0) {
               stats.backtracks++;
               return false;
            }
            
            var tries = Math.min(MAX_TRIES_PER_SLOT, N);
            for (var t = 0; t < tries; t++) {
               double r          = rng.nextFloat();
               int    idxInArray = (int) (r * r * r * N);
               var    w          = entry.words[idxInArray];
               var    lemIdx     = Lemma.unpackIndex(w);
               if (used.get(lemIdx)) continue;
               
               if (!placeWord(grid, s, w, ctx.undo, depth)) continue;
               
               used.set(lemIdx);
               assigned[k] = w;
               
               if (backtrack(depth + 1)) return true;
               
               assigned[k] = X;
               used.clear(lemIdx);
               grid.undoPlace(ctx.undo[depth << 1], ctx.undo[(depth << 1) | 1]);
            }
            
            stats.backtracks++;
            return false;
         }
      }
      
      // initial render (same feel)
      var solver = new Solver();
      solver.renderProgress();
      var ok = solver.backtrack(0);
      // final progress line
      if (!multiThreaded) {
         System.out.print("\r" + Strings.padRight("", 120) + "\r");
         System.out.flush();
      }
      
      stats.seconds = (System.currentTimeMillis() - t0) / 1000.0;
      Map<Integer, Long> lemmaMap = new HashMap<>();
      for (var i = 0; i < assigned.length; i++) {
         if (assigned[i] != X) {
            lemmaMap.put(i, assigned[i]);
         }
      }
      var res = new FillResult(ok, new Gridded(grid), lemmaMap, stats);
      
      // print a final progress line
      if (Main.VERBOSE && !multiThreaded) {
         System.out.println(
                 String.format(Locale.ROOT,
                               "[######################] %d/%d slots | nodes=%d | backtracks=%d | mrv=%d | %.1fs",
                               lemmaMap.size(), TOTAL, stats.nodes, stats.backtracks, stats.lastMRV, stats.seconds
                              )
                           );
      }
      
      return res;
   }
}