package puzzle;

import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.experimental.Accessors;
import lombok.experimental.Delegate;
import lombok.val;
import precomp.Neighbors9x8;
import precomp.Neighbors9x8.rci;
import puzzle.Export.Bit1029;
import puzzle.Export.DictEntryDTO;
import puzzle.Export.Gridded;
import puzzle.Export.Strings;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Locale;
import java.util.stream.IntStream;
import static java.nio.charset.StandardCharsets.*;

/**
 * NOTE:
 * A) generate randoms for and idx and direction (clue together)
 * B) convert IDX_PATH_0_BASE into bit-pattern (i think) Length should not be checked, just left empty if not good.
 * C) store more information in the byte[] even consider the long[] or try sqeeze the 288 options (clue) in a byte.
 * D) separate clue into three long pairs of hi and lo, (v,h,r) so we do not store the clue anymore in the grid array at all
 * E) store letter-set also in a long pair so we can use it in path determinations
 * F) pre-determine random clue arrangements, so they do not involve impossible clue's such as bottom at last or the line before that and such to all sides
 * G) Check 3wall, the current implementation may be faster, but the accuracy is lower, degrade performance on the filler
 */

/**
 * SwedishGenerator.java
 *
 * Usage:
 *   javac SwedishGenerator.java
 *   java SwedishGenerator [--seed N] [--pop N] [--gens N] [--tries N] [--words word-list.txt]
 */
@SuppressWarnings("ALL")
public record SwedishGenerator(Rng rng, int[] stack, Clues cache) {
   
   record CandidateInfo(int[] indices, int count) { }
   
   //@formatter:off
   @FunctionalInterface interface SlotVisitor { void visit(int key, long lo, long hi); }
   //@formatter:on
   static final long   GT_1_OFFSET_53_BIT       = 0x3E00000000000000L;
   static final long   X                        = 0L;
   static final int    LOG_EVERY_MS             = 200;
   static final int    BAR_LEN                  = 22;
   static final int    C                        = Config.PUZZLE_COLS;
   static final double CROSS_R                  = (C - 1) / 2.0;
   static final int    R                        = Config.PUZZLE_ROWS;
   static final double CROSS_C                  = (R - 1) / 2.0;
   static final int    SIZE                     = C * R;// ~18
   static final int    SIZE_MIN_1               = SIZE - 1;// ~18
   static final double SIZED                    = (double) SIZE;// ~18
   static final int    TARGET_CLUES             = SIZE >>> 2;
   static final int    MAX_WORD_LENGTH          = C <= R ? C : R;
   static final int    MAX_WORD_LENGTH_PLUS_ONE = MAX_WORD_LENGTH + 1;
   static final int    MIN_LEN                  = Config.MIN_LEN;
   static final int    MAX_TRIES_PER_SLOT       = Config.MAX_TRIES_PER_SLOT;
   static final int    STACK_SIZE               = 64;
   static final char   C_DASH                   = '\0';
   static final byte   _1                       = 49, _9 = 57, A = 65, Z = 90, DASH = (byte) C_DASH;
   //72 << 3;
   static final int CLUE_INDEX_MAX_SIZE = (288 | 3) + 1;
   static int clamp(int x, int a, int b) { return Math.max(a, Math.min(b, x)); }
   @AllArgsConstructor
   static class Pick {
      
      Slot  slot;
      int[] indices;
      int   count;
   }
   
   // 0b11
   //0b00
   // 0b01
   // 0b10
   
   static final byte    B0             = (byte) 0;
   static final byte    B64            = (byte) 64;
   static final long[]  OFFSETS_D_IDX  = Neighbors9x8.OFFSET_D_IDX_0_BASE;
   static final rci[]   IT             = Neighbors9x8.IT;
   static final int[][] MUTATE_RI      = new int[SIZE][625];
   static final long[]  NBR8_PACKED_LO = Neighbors9x8.NBR8_PACKED_LO;
   static final long[]  NBR8_PACKED_HI = Neighbors9x8.NBR8_PACKED_HI;
   static final long[]  PATH_LO        = Neighbors9x8.PATH_LO;
   static final long[]  PATH_HI        = Neighbors9x8.PATH_HI;
   
   static {
      for (int i = 0; i < SIZE; i++) {
         int k = 0;
         for (int dr1 = -2; dr1 <= 2; dr1++)
            for (int dr2 = -2; dr2 <= 2; dr2++)
               for (int dc1 = -2; dc1 <= 2; dc1++)
                  for (int dc2 = -2; dc2 <= 2; dc2++) {
                     val ti = IT[i];
                     MUTATE_RI[i][k++] = Grid.offset(clamp(ti.r() + dr1 + dr2, 0, R - 1),
                                                     clamp(ti.c() + dc1 + dc2, 0, C - 1));
                  }
      }
   }
   
   static final Pick PICK_DONE     = null;//new Pick(null, null, 0, true);
   static final Pick PICK_NOT_DONE = new Pick(null, null, 0);
   
   @RequiredArgsConstructor
   @Getter
   @Accessors(fluent = true)
   public static final class FillStats {
      
      final public long   nodes;
      final public long   backtracks;
      final public double seconds;
      final public int    lastMRV;
      public       double simplicity;
   }
   
   public static record FillResult(boolean ok,
                                   Gridded grid,
                                   long[] clueMap,
                                   @Delegate FillStats stats) {
      
      public void calcSimpel() {
         if (ok) {
            int k = 0;
            for (var n = 1; n < clueMap.length; n++) {
               if (clueMap[n] != 0L) {
                  k++;
                  stats.simplicity += Lemma.simpel(clueMap[n]);
               }
            }
            stats.simplicity = k == 0 ? 0 : stats.simplicity / k;
         }
      }
      public int wordCount() {
         int k = 0;
         for (var n = 1; n < clueMap.length; n++) {
            if (clueMap[n] != 0L) {
               k++;
            }
         }
         return k;
      }
   }
   
   static final class Rng {
      
      @Getter private int x;
      Rng(int seed) {
         var s = seed;
         if (s == 0) s = 1;
         this.x = s;
      }
      int nextU32() {
         var y = x;
         y ^= (y << 13);
         y ^= (y >>> 17);
         y ^= (y << 5);
         x = y;
         return y;
      }
      byte randbyte(int min, int max) {
         var u     = (nextU32() & 0xFFFFFFFFL);
         var range = (long) max - (long) min + 1L;
         return (byte) (min + (u % range));
      }
      int randint2bit()      { return nextU32() & 3; }
      byte randint2bitByte() { return (byte) (nextU32() & 3); }
      int randint(int min, int max) {
         var u     = (nextU32() & 0xFFFFFFFFL);
         var range = (long) max - (long) min + 1L;
         return (int) (min + (u % range));
      }
      double nextFloat() { return (nextU32() & 0xFFFFFFFFL) / 4294967295.0; }
      int biasedIndexPow3(int N) {
         int m = Math.min(nextU32(), Math.min(nextU32(), nextU32()));
         return (int) (((m & 0xFFFFFFFFL) * (long) N) >>> 32);
      }
   }
   
   @AllArgsConstructor
   static class Clues {
      
      long lo, hi, vlo, vhi, rlo, rhi;
      public static Clues createEmpty() {
         return new Clues(0, 0, 0, 0, 0, 0);
      }
      public Clues deepCopyGrid() { return new Clues(lo, hi, vlo, vhi, rlo, rhi); }
      boolean clueless(int idx) {
         if (!isClue(idx)) return false;
         clearClue(idx);
         return true;
      }
      public boolean hasRoomForClue(long packed) { return (packed) != X && notClue(packed & 0x7FL) && notClue((packed >>> 7) & 0x7FL); }
      
      public void setClue(int ri, byte idx) {
         if ((ri & 64) == 0) {
            long mask = 1L << ri;
            lo |= mask;
            if ((idx & 1) != 0) vlo |= mask;
            else vlo &= ~mask;
            if ((idx & 2) != 0) rlo |= mask;
            else rlo &= ~mask;
         } else {
            long mask = 1L << (ri & 63);
            hi |= mask;
            if ((idx & 1) != 0) vhi |= mask;
            else vhi &= ~mask;
            if ((idx & 2) != 0) rhi |= mask;
            else rhi &= ~mask;
         }
      }
      public byte digitAt(int idx) {
         if ((idx & 64) == 0) {
            int v = (int) ((vlo >>> idx) & 1L);
            int r = (int) ((rlo >>> idx) & 1L);
            return (byte) ((r << 1) | v);
         } else {
            int v = (int) ((vhi >>> (idx & 63)) & 1L);
            int r = (int) ((rhi >>> (idx & 63)) & 1L);
            return (byte) ((r << 1) | v);
         }
      }
      public void clearClue(int idx) {
         if ((idx & 64) == 0) {
            long mask = ~(1L << idx);
            lo &= mask;
            vlo &= mask;
            rlo &= mask;
         } else {
            long mask = ~(1L << (idx & 63));
            hi &= mask;
            vhi &= mask;
            rhi &= mask;
         }
      }
      public boolean isClue(long index)  { return ((index & 64) == 0) ? ((lo >>> index) & 1L) != X : ((hi >>> (index & 63)) & 1L) != X; }
      public boolean isClue(int index)   { return ((index & 64) == 0) ? ((lo >>> index) & 1L) != X : ((hi >>> (index & 63)) & 1L) != X; }
      public boolean notClue(long index) { return ((index & 64) == 0) ? ((lo >>> index) & 1L) == X : ((hi >>> (index & 63)) & 1L) == X; }
      public boolean notClue(int index)  { return ((index & 64) == 0) ? ((lo >>> index) & 1L) == X : ((hi >>> (index & 63)) & 1L) == X; }
      
      public int clueCount()             { return Long.bitCount(lo) + Long.bitCount(hi); }
      public double similarity(Clues b) {
         long matchLo = (~(lo ^ b.lo)) & (~lo | (~(vlo ^ b.vlo) & ~(rlo ^ b.rlo)));
         long matchHi = (~(hi ^ b.hi)) & (~hi | (~(vhi ^ b.vhi) & ~(rhi ^ b.rhi)));
         
         long maskLo = (SIZE >= 64) ? -1L : (1L << SIZE) - 1;
         long maskHi = (SIZE <= 64) ? 0L : (1L << (SIZE - 64)) - 1;
         
         return (Long.bitCount(matchLo & maskLo) + Long.bitCount(matchHi & maskHi)) / SIZED;
         
         
       /*  var same = 0;
         for (int i = 0; i < SIZE; i++) if (digitAt(i) == b.digitAt(i)) same++;
         return same / SIZED;*/
      }
      public Grid toGrid() { return new Grid(new byte[SIZE], lo, hi); }
      public void forEachSlot(SlotVisitor visitor) {
         for (var l = lo; l != X; l &= l - 1) processSlot(this, visitor, Long.numberOfTrailingZeros(l));
         for (var h = hi; h != X; h &= h - 1) processSlot(this, visitor, 64 | Long.numberOfTrailingZeros(h));
      }
      public Clues from(Clues best) {
         lo  = best.lo;
         hi  = best.hi;
         vlo = best.vlo;
         vhi = best.vhi;
         rlo = best.rlo;
         rhi = best.rhi;
         return this;
      }
   }
   
   @AllArgsConstructor
   static class Grid {
      
      final  byte[] g;
      public long   lo, hi;
      static int offset(int r, int c) { return r | (c << 3); }
      /// the pos will never target a clue
      public byte letter32At(int pos) { return g[pos]; }
      void setLetterLo(int idx, byte ch) {
         lo |= (1L << idx);
         
         g[idx] = ch;
      }
      void setLetterHi(int idx, byte ch) {
         hi |= (1L << (idx & 63));
         
         g[idx] = ch;
      }
      void setLetter(int idx, byte ch) {
         if ((idx & 64) == 0)
            setLetterLo(idx, ch);
         else
            setLetterHi(idx, ch);
      }
      
      private void clearletterLo(int idx) {
         g[idx] = DASH;
         
         lo &= ~(1L << idx);
      }
      private void clearletterHi(int idx) {
         g[idx] = DASH;
         
         hi &= ~(1L << (idx & 63));
      }
      void undoPlace(long maskLo, long maskHi) {
         for (long b = maskLo; b != 0; b &= b - 1) clearletterLo(Long.numberOfTrailingZeros(b));
         for (long b = maskHi; b != 0; b &= b - 1) clearletterHi(64 | Long.numberOfTrailingZeros(b));
      }
   }
   
   static record DictEntry(long[] words, long[][] posBitsets, int length, int numlong) { }
   
   public static interface Lemma {
      
      static final long LETTER_MASK = (1L << 40) - 1;     // low 40 bits
      static final long INDEX_MASK  = (1L << 24) - 1;     // 24 bits
      
      static long pack(String word)         { return pack(word.getBytes(US_ASCII)); }
      static long pack(int index, byte[] b) { return pack(b) | ((long) index << 40); }
      static long pack(byte[] b) {
         long w = 0;
         for (var i = 0; i < b.length; i++) w |= ((long) b[i] & 31) << (i * 5);
         return w;
      }
      static public long from(int index, String word) { return pack(index, word.getBytes(US_ASCII)); }
      static byte byteAt(long word, int idx)          { return (byte) ((word >>> (idx * 5)) & 0b11111); }
      static String[] clue(long w)                    { return CsvIndexService.clues(unpackIndex(w)); }
      static int simpel(long w)                       { return CsvIndexService.simpel(unpackIndex(w)); }
      static int length(long word)                    { return ((63 - Long.numberOfLeadingZeros(word & LETTER_MASK)) / 5) + 1; }
      public static String asWord(long word) {
         var b = new byte[Lemma.length(word)];
         for (var i = 0; i < b.length; i++) b[i] = (byte) ((word >>> (i * 5)) & 0b11111 | B64);
         return new String(b, US_ASCII);
      }
      static int unpackIndex(long w)   { return (int) (w >>> 40); }
      static int unpackLetters(long w) { return (int) (w & LETTER_MASK); }
   }
   
   public static record Dict(
           DictEntry[] index,
           int length) {
      
      public Dict(long[] wordz) {
         var index = new DictEntryDTO[MAX_WORD_LENGTH_PLUS_ONE];
         Arrays.setAll(index, i -> new DictEntryDTO(i));
         for (var lemma : wordz) {
            var L = Lemma.length(lemma);
            
            var entry = index[L];
            var idx   = entry.words().size();
            entry.words().add(lemma);
            
            for (var i = 0; i < L; i++) {
               var letter = Lemma.byteAt(lemma, i) - 1;
               if (letter < 0 || letter >= 26) throw new RuntimeException("Illegal letter: " + letter + " in word " + lemma);
               entry.pos()[i][letter].add(idx);
            }
         }
         for (int i = MIN_LEN; i < index.length; i++) if (index[i].words().size() <= 0) throw new RuntimeException("No words for length " + i);
         this(Arrays.stream(index).map(i -> {
                 var words    = i.words().toArray();
                 int numWords = words.length;
                 int numLongs = (numWords + 63) >>> 6;
                 var bitsets  = new long[i.pos().length * 26][numLongs];
                 for (int p = 0; p < i.pos().length; p++) {
                    for (int l = 0; l < 26; l++) {
                       var list = i.pos()[p][l];
                       var bs   = bitsets[p * 26 + l];
                       for (int k = 0; k < list.size(); k++) {
                          int wordIdx = list.data()[k];
                          bs[wordIdx >>> 6] |= (1L << (wordIdx & 63));
                       }
                    }
                 }
                 return new DictEntry(words, bitsets, words.length, (words.length + 63) >>> 6);
              }).toArray(DictEntry[]::new),
              Arrays.stream(index).mapToInt(i -> i.words().size()).sum());
      }
      static Dict loadDict(String wordsPath) {
         try {
            var map = new LongArrayList(100_000);
            Files.lines(Path.of(wordsPath), UTF_8).forEach(line -> CsvIndexService.lineToLemma(line, map::add));
            return new Dict(map.toArray());
         } catch (IOException e) {
            e.printStackTrace();
            throw new RuntimeException("Failed to load dictionary from " + wordsPath, e);
         }
      }
   }
   
   static record Slot(int key, long lo, long hi) {
      
      static final int BIT_FOR_DIR = 2;
      static Slot from(int key, long lo, long hi)   { return new Slot(key, lo, hi); }
      
      public int length()                           { return Long.bitCount(lo) + Long.bitCount(hi); }
      public int clueIndex()                        { return clueIndex(key); }
      public static int clueIndex(int key)          { return key >>> BIT_FOR_DIR; }
      public static int dir(int key)                { return key & 3; }
      public boolean horiz()                        { return horiz(key); }
      public boolean increasing()                   { return (key & 2) == 0; }
      public static boolean increasing(int dir)     { return (dir & 2) == 0; }
      public IntStream walk()                       { return Gridded.walk((byte) key, lo, hi); }
      public static boolean horiz(int d)            { return (d & 1) != 0; }
      public static int packSlotDir(int idx, int d) { return (idx << BIT_FOR_DIR) | d; }
   }
   
   private static void processSlot(Clues grid, SlotVisitor visitor, int idx) {
      int key = Slot.packSlotDir(idx, grid.digitAt(idx));         // 0..3
      
      long rayLo = PATH_LO[key];
      long rayHi = PATH_HI[key];
      
      // only consider clue cells
      long hitsLo = rayLo & grid.lo;
      long hitsHi = rayHi & grid.hi;
      
      // slice ray to stop before first clue, depending on direction monotonicity
      // right/down => increasing indices; up/left => decreasing indices
      
      if (Slot.increasing(key)) {
         // first clue is lowest index among hits (lo first, then hi)
         if (hitsLo != 0) {
            long stop = 1L << Long.numberOfTrailingZeros(hitsLo);
            rayLo &= (stop - 1);
            rayHi = 0; // any hi is beyond the stop
         } else if (hitsHi != 0) {
            long stop = 1L << Long.numberOfTrailingZeros(hitsHi);
            // keep all lo (lo indices are < any hi index), but cut hi below stop
            rayHi &= (stop - 1);
         }
      } else {
         // first clue is highest index among hits (hi first, then lo)
         if (hitsHi != 0) {
            int  msb  = 63 - Long.numberOfLeadingZeros(hitsHi);
            long stop = 1L << msb;
            rayHi &= ~((stop << 1) - 1); // keep bits > stop
            rayLo = 0;                   // lo indices are below stop
         } else if (hitsLo != 0) {
            int  msb  = 63 - Long.numberOfLeadingZeros(hitsLo);
            long stop = 1L << msb;
            rayLo &= ~((stop << 1) - 1);
         }
      }
      
      visitor.visit(key, rayLo, rayHi);
      //if ((rayLo | rayHi) == 0L) throw new RuntimeException()
   }
   
   static Slot[] extractSlots(Clues grid) {
      var   slots = new Slot[grid.clueCount()];
      int[] N     = new int[]{ 0 };
      grid.forEachSlot((key, lo, hi) -> slots[N[0]++] = Slot.from(key, lo, hi));
      return slots;
   }
   
   ///  does not modify the grid
   long maskFitness(final Clues grid) {
      
      long    cHLo     = 0L, cHHi = 0L, cVLo = 0L, cVHi = 0L;
      long    lo_cl    = grid.lo, hi_cl = grid.hi;
      long    penalty  = (((long) Math.abs(grid.clueCount() - TARGET_CLUES)) * 16000L);
      boolean hasSlots = false;
      
      for (int i = 0; i < 65; i += 64) {
         for (long bits = (i == 0 ? lo_cl : hi_cl); bits != X; bits &= bits - 1) {
            int clueIdx = i | Long.numberOfTrailingZeros(bits);
            int key     = Slot.packSlotDir(clueIdx, grid.digitAt(clueIdx));
            ///  long clueBits = (i == 0 ? lo_cl : hi_cl);
            ///  long vBits    = (i == 0 ? grid.vlo : grid.vhi);
            ///  long rBits    = (i == 0 ? grid.rlo : grid.rhi);
            ///  for (long bits = clueBits; bits != X; bits &= bits - 1) {
            ///     long lsb     = bits & -bits;
            ///     int  clueIdx = i | Long.numberOfTrailingZeros(lsb);
            ///     int  v       = (vBits & lsb) != 0 ? 1 : 0;
            ///     int  r       = (rBits & lsb) != 0 ? 1 : 0;
            ///     int  key     = Slot.packSlotDir(clueIdx, (r << 1) | v);
            long rLo = PATH_LO[key], rHi = PATH_HI[key];
            long hLo = rLo & lo_cl, hHi = rHi & hi_cl;
            if (Slot.increasing(key)) {
               if (hLo != 0) {
                  rLo &= ((1L << Long.numberOfTrailingZeros(hLo)) - 1);
                  rHi = 0;
               } else if (hHi != 0) { rHi &= ((1L << Long.numberOfTrailingZeros(hHi)) - 1); }
            } else {
               if (hHi != 0) {
                  int msb = 63 - Long.numberOfLeadingZeros(hHi);
                  rHi &= ~((1L << msb << 1) - 1);
                  rLo = 0;
               } else if (hLo != 0) {
                  int msb = 63 - Long.numberOfLeadingZeros(hLo);
                  rLo &= ~((1L << msb << 1) - 1);
               }
            }
            if ((rLo | rHi) != 0L) {
               hasSlots = true;
               if (Slot.horiz(key)) {
                  cHLo |= rLo;
                  cHHi |= rHi;
               } else {
                  cVLo |= rLo;
                  cVHi |= rHi;
               }
               if ((Long.bitCount(rLo) + Long.bitCount(rHi)) < MIN_LEN) penalty += 8000;
            } else {
               penalty += 25000;
            }
         }
      }
      
      if (!hasSlots) return 1_000_000_000L;
      long seenLo = 0L, seenHi = 0L;

// loop over beide helften
      for (int base = 0; base <= 64; base += 64) {
         long clueMask = (base == 0) ? lo_cl : hi_cl;
         long seenMask = (base == 0) ? seenLo : seenHi;
         
         // "unseen clues" in deze helft
         for (long bits = clueMask & ~seenMask; bits != 0L; bits &= bits - 1) {
            int clueIdx = base | Long.numberOfTrailingZeros(bits);
            
            // start nieuwe component
            int size = 0;
            int sp   = 0;
            stack[sp++] = clueIdx;
            
            // mark seen
            if ((clueIdx & 64) == 0) seenLo |= 1L << clueIdx;
            else seenHi |= 1L << (clueIdx & 63);
            
            // flood fill / bfs
            while (sp > 0) {
               int cur = stack[--sp];
               size++;
               
               // neighbors als 2x long masks
               long nLo = NBR8_PACKED_LO[cur];
               long nHi = NBR8_PACKED_HI[cur];
               
               // filter: alleen clues, en nog niet seen
               nLo &= lo_cl & ~seenLo;
               nHi &= hi_cl & ~seenHi;
               
               // push lo-neighbors
               while (nLo != 0L) {
                  long lsb  = nLo & -nLo;
                  int  nidx = Long.numberOfTrailingZeros(nLo); // 0..63
                  seenLo |= lsb;
                  
                  stack[sp++] = nidx;
                  
                  nLo &= nLo - 1;
               }
               
               // push hi-neighbors
               while (nHi != 0L) {
                  long lsb  = nHi & -nHi;
                  int  nidx = 64 | Long.numberOfTrailingZeros(nHi); // 64..127
                  seenHi |= lsb;
                  
                  stack[sp++] = nidx;
                  
                  nHi &= nHi - 1;
               }
            }
            
            if (size >= 2) penalty += (size - 1L) * 120L;
         }
      }
      
      for (long bits = ~lo_cl; bits != X; bits &= bits - 1) {
         int clueIdx = Long.numberOfTrailingZeros(bits);
         var rci     = IT[clueIdx];
         if ((4 - rci.nbrCount()) + Long.bitCount(rci.n1() & lo_cl) + Long.bitCount(rci.n2() & hi_cl) >= 3) penalty += 400;
         boolean h = (cHLo & (1L << clueIdx)) != 0;
         boolean v = (cVLo & (1L << clueIdx)) != 0;
         if (!h && !v) penalty += 1500;
         else if (h && v) { /* ok */ } else if (h | v) penalty += 200;
         else penalty += 600;
      }
      for (long bits = ~hi_cl & 0xFFL; bits != X; bits &= bits - 1) {
         int clueIdx = Long.numberOfTrailingZeros(bits);
         var rci     = IT[64 | clueIdx];
         if ((4 - rci.nbrCount()) + Long.bitCount(rci.n1() & lo_cl) + Long.bitCount(rci.n2() & hi_cl) >= 3) penalty += 400;
         boolean h = (cHHi & (1L << clueIdx)) != 0;
         boolean v = (cVHi & (1L << clueIdx)) != 0;
         if (!h && !v) penalty += 1500;
         else if (h && v) { /* ok */ } else if (h | v) penalty += 200;
         else penalty += 600;
      }
      
      return penalty;
   }
   
   Clues randomMask() {
      var g = Clues.createEmpty();
      for (int placed = 0, guard = 0, idx; placed < TARGET_CLUES && guard < 4000; guard++) {
         idx = rng.randint(0, SIZE_MIN_1);
         if (g.isClue(idx)) continue;
         var d_idx = rng.randint2bitByte();
         if (g.hasRoomForClue(OFFSETS_D_IDX[Slot.packSlotDir(idx, d_idx)])) {
            g.setClue(idx, d_idx);
            placed++;
         }
      }
      return g;
   }
   Clues mutate(Clues c) {
      int ri;
      var bytes = MUTATE_RI[rng.randint(0, SIZE_MIN_1)];
      for (var k = 0; k < 4; k++) {
         ri = bytes[rng.randint(0, 624)];
         if (!c.clueless(ri)) {
            var d_idx = rng.randint2bitByte();
            if (c.hasRoomForClue(OFFSETS_D_IDX[Slot.packSlotDir(ri, d_idx)])) c.setClue(ri, d_idx);
         }
      }
      return c;
   }
   Clues crossover(Clues a, Clues other) {
      var out   = a.deepCopyGrid();
      var theta = rng.nextFloat() * Math.PI;
      var nc    = Math.cos(theta);
      var nr    = Math.sin(theta);
      
      for (var rci : IT) {
         int i = rci.i();
         if ((rci.cross_r()) * nc + (rci.cross_c()) * nr < 0) {
            var ch = other.digitAt(i);
            if (out.digitAt(i) != ch) {
               if (other.isClue(i)) {
                  out.setClue(i, ch);
               } else {
                  out.clearClue(i);
               }
            }
         }
      }
      
      // long maskLo = 0, maskHi = 0;
      // for (var rci : IT) {
      //    if ((rci.cross_r()) * nc + (rci.cross_c()) * nr < 0) {
      //       int i = rci.i();
      //       if (i < 64) maskLo |= (1L << i);
      //       else maskHi |= (1L << (i - 64));
      //    }
      // }
      //
      // out.lo = (out.lo & ~maskLo) | (other.lo & maskLo);
      // out.hi = (out.hi & ~maskHi) | (other.hi & maskHi);
      // out.vlo = (out.vlo & ~maskLo) | (other.vlo & maskLo);
      // out.vhi = (out.vhi & ~maskHi) | (other.vhi & maskHi);
      // out.rlo = (out.rlo & ~maskLo) | (other.rlo & maskLo);
      // out.rhi = (out.rhi & ~maskHi) | (other.rhi & maskHi);
      
      for (var lo = out.lo; lo != X; lo &= lo - 1L) clearClues(out, Long.numberOfTrailingZeros(lo));
      for (var hi = out.hi; hi != X; hi &= hi - 1L) clearClues(out, 64 | Long.numberOfTrailingZeros(hi));
      return out;
   }
   public static void clearClues(Clues out, int idx) { if (!out.hasRoomForClue(OFFSETS_D_IDX[Slot.packSlotDir(idx, out.digitAt(idx))])) out.clearClue(idx); }
   
   Clues hillclimb(Clues start, int limit) {
      var best  = start;
      var bestF = maskFitness(best);
      var fails = 0;
      
      while (fails < limit) {
         cache.from(best);
         var cand = mutate(best);
         var f    = maskFitness(cand);
         if (f < bestF) {
            best  = cand;
            bestF = f;
            fails = 0;
         } else {
            best.from(cache);
            fails++;
         }
      }
      return best;
   }
   
   public Clues generateMask(int popSize, int gens, int pairs) {
      class GridAndFit {
         
         Clues grid;
         long  fite = -1;
         GridAndFit(Clues grid) { this.grid = grid; }
         long fit() {
            if (fite == -1) this.fite = maskFitness(grid);
            return this.fite;
         }
      }
      if (Main.VERBOSE) System.out.println("generateMask init pop: " + popSize);
      var pop = new ArrayList<GridAndFit>();
      for (var i = 0; i < popSize; i++) {
         pop.add(new GridAndFit(hillclimb(randomMask(), 180)));
      }
      
      for (var gen = 0; gen < gens; gen++) {
         if (Thread.currentThread().isInterrupted()) break;
         var children = new ArrayList<GridAndFit>();
         
         for (var k = 0; k < pairs; k++) {
            var p1    = pop.get(rng.randint(0, pop.size() - 1));
            var p2    = pop.get(rng.randint(0, pop.size() - 1));
            var child = crossover(p1.grid, p2.grid);
            children.add(new GridAndFit(hillclimb(child, 70)));
         }
         
         pop.addAll(children);
         pop.sort(Comparator.comparingLong(GridAndFit::fit));
         
         var next = new ArrayList<GridAndFit>();
         for (var cand : pop) {
            if (next.size() >= popSize) break;
            var ok = true;
            for (var kept : next) {
               if (cand.grid.similarity(kept.grid) > 0.92) {
                  ok = false;
                  break;
               }
            }
            if (ok) next.add(cand);
         }
         pop = next;
         
         if (Main.VERBOSE && (gen & 15) == 15) System.out.println("  gen " + gen + "/" + gens + " bestFitness=" + pop.get(0).fit());
      }
      GridAndFit best = pop.get(0);
      for (int i = 1; i < pop.size(); i++) {
         var x = pop.get(i);
         if (x.fit() < best.fit()) best = x;
      }
      return best.grid;
   }
   
   static long patternForSlot(Grid grid, Slot s) {
      if ((s.lo & grid.lo) == 0 && (s.hi & grid.hi) == 0) {
         return 0;
      }
      long p = 0;
      if (s.increasing()) {
         for (long b = s.lo & grid.lo; b != 0; b &= b - 1) {
            int  idx = Long.numberOfTrailingZeros(b);
            byte val = grid.g[idx];
            int  i   = Long.bitCount(s.lo & ((1L << idx) - 1));
            p |= ((long) (i * 26 + val)) << (i << 3);
         }
         int offset = Long.bitCount(s.lo);
         for (long b = s.hi & grid.hi; b != 0; b &= b - 1) {
            int  idx = Long.numberOfTrailingZeros(b);
            byte val = grid.g[64 | idx];
            int  i   = offset + Long.bitCount(s.hi & ((1L << idx) - 1));
            p |= ((long) (i * 26 + val)) << (i << 3);
         }
      } else {
         int offset = Long.bitCount(s.hi);
         for (long b = s.hi & grid.hi; b != 0; b &= b - 1) {
            int  idx = Long.numberOfTrailingZeros(b);
            byte val = grid.g[64 | idx];
            int  i   = Long.bitCount(s.hi & ~((1L << idx) | ((1L << idx) - 1)));
            p |= ((long) (i * 26 + val)) << (i << 3);
         }
         for (long b = s.lo & grid.lo; b != 0; b &= b - 1) {
            int  idx = Long.numberOfTrailingZeros(b);
            byte val = grid.g[idx];
            int  i   = offset + Long.bitCount(s.lo & ~((1L << idx) | ((1L << idx) - 1)));
            p |= ((long) (i * 26 + val)) << (i << 3);
         }
      }
      return p;
   }
   static int slotScore(byte[] count, Slot s) {
      int cross = 0;
      for (long b = s.lo; b != 0; b &= b - 1) cross += (count[Long.numberOfTrailingZeros(b)] - 1);
      for (long b = s.hi; b != 0; b &= b - 1) cross += (count[64 | Long.numberOfTrailingZeros(b)] - 1);
      return cross * 10 + s.length();
   }
   static boolean placeWord(Grid grid, Slot s, long w, long[] undoBuffer, int offset) {
      long maskLo = 0, maskHi = 0;
      int  i      = 0;
      if (s.increasing()) {
         for (long b = s.lo; b != 0; b &= b - 1, i++) {
            int  idx = Long.numberOfTrailingZeros(b);
            byte cur = grid.letter32At(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskLo |= (1L << idx);
               grid.setLetterLo(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
         }
         for (long b = s.hi; b != 0; b &= b - 1, i++) {
            int  idx = 64 | Long.numberOfTrailingZeros(b);
            byte cur = grid.letter32At(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskHi |= (1L << (idx & 63));
               grid.setLetterHi(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
         }
      } else {
         for (long b = s.hi; b != 0; i++) {
            int  msb = 63 - Long.numberOfLeadingZeros(b);
            int  idx = 64 | msb;
            byte cur = grid.letter32At(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskHi |= (1L << msb);
               grid.setLetterHi(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
            b &= ~(1L << msb);
         }
         for (long b = s.lo; b != 0; i++) {
            int  msb = 63 - Long.numberOfLeadingZeros(b);
            int  idx = msb;
            byte cur = grid.letter32At(idx), ch = Lemma.byteAt(w, i);
            if (cur == DASH) {
               maskLo |= (1L << msb);
               grid.setLetterLo(idx, ch);
            } else if (cur != ch) {
               grid.undoPlace(maskLo, maskHi);
               return false;
            }
            b &= ~(1L << msb);
         }
      }
      undoBuffer[offset << 1]       = maskLo;
      undoBuffer[(offset << 1) | 1] = maskHi;
      return true;
   }
   
   static int[] candidateInfoForPattern(long[] res, long pattern, long[][] posBitsets, int numLongs) {
      boolean first = true;
      
      for (long p = pattern; p != 0; ) {
         int combined = (int) (p & 0xFF);
         if (combined != 0) {
            long[] bs = posBitsets[combined - 1];
            if (first) {
               System.arraycopy(bs, 0, res, 0, numLongs);
               first = false;
            } else {
               for (int k = 0; k < numLongs; k++) res[k] &= bs[k];
            }
            p >>>= 8;
         } else {
            p >>>= (Long.numberOfTrailingZeros(p) & ~7);
         }
      }
      
      int count = 0;
      for (int k = 0; k < numLongs; k++) count += Long.bitCount(res[k]);
      
      int[] indices = new int[count];
      int   ki      = 0;
      for (int k = 0; k < numLongs; k++) {
         long w = res[k];
         while (w != 0) {
            int t = Long.numberOfTrailingZeros(w);
            indices[ki++] = (k << 6) | t;
            w &= w - 1;
         }
      }
      
      return indices;
   }
   
   static int candidateCountForPattern(final long[] res, final long pattern, final long[][] posBitsets, final int numLongs) {
      boolean first = true;
      
      for (long p = pattern; p != 0; ) {
         int combined = (int) (p & 0xFF);
         if (combined != 0) {
            long[] bs = posBitsets[combined - 1];
            if (first) {
               System.arraycopy(bs, 0, res, 0, numLongs);
               first = false;
            } else {
               for (int k = 0; k < numLongs; k++) res[k] &= bs[k];
            }
            p >>>= 8;
         } else {
            p >>>= (Long.numberOfTrailingZeros(p) & ~7);
         }
      }
      
      int count = 0;
      for (int k = 0; k < numLongs; k++) count += Long.bitCount(res[k]);
      return count;
   }
   
   static void scoreSlots(int[] slotScores, Slot[] slots) {
      val count = new byte[SIZE];
      for (var s : slots) {
         for (long b = s.lo; b != 0; b &= b - 1) count[Long.numberOfTrailingZeros(b)]++;
         for (long b = s.hi; b != 0; b &= b - 1) count[64 | Long.numberOfTrailingZeros(b)]++;
      }
      for (int i = 0; i < slots.length; i++) slotScores[i] = slotScore(count, slots[i]);
      
   }
   public static FillResult fillMask(Rng rng, Slot[] slots, Grid mask, DictEntry[] dictIndex) {
      val multiThreaded = Thread.currentThread().getName().contains("pool");
      val NO_LOG        = (!Main.VERBOSE || multiThreaded);
      val grid          = mask;
      val used          = new Bit1029();
      val assigned      = new long[CLUE_INDEX_MAX_SIZE];
      val bitset        = new long[2500];
      val undo          = new long[64];
      
      val TOTAL      = slots.length;
      val slotScores = new int[TOTAL];
      
      scoreSlots(slotScores, slots);
      
      val t0 = System.currentTimeMillis();
      
      class Solver {
         
         private final Pick CARRIER = new Pick(null, null, 0);
         long nodes;
         long backtracks;
         int  lastMRV;
         long lastLog = t0;
         Pick current = CARRIER;
         void renderProgress() {
            var now = System.currentTimeMillis();
            if ((now - lastLog) < LOG_EVERY_MS) return;
            lastLog = (now);
            var done = 0;
            for (var lemma : assigned) {
               if (lemma != X) done++;
            }
            var pct     = (TOTAL == 0) ? 100 : (int) Math.floor((done / (double) TOTAL) * 100);
            var filled  = Math.min(BAR_LEN, (int) Math.floor((pct / 100.0) * BAR_LEN));
            var bar     = "[" + "#".repeat(filled) + "-".repeat(BAR_LEN - filled) + "]";
            var elapsed = String.format(Locale.ROOT, "%.1fs", (now - t0) / 1000.0);
            
            var msg = String.format(
                    Locale.ROOT,
                    "%s %d/%d slots | nodes=%d | backtracks=%d | mrv=%d | %s",
                    bar, done, TOTAL, nodes, backtracks, lastMRV, elapsed
                                   );
            System.out.print("\r" + Strings.padRight(msg, 120));
            System.out.flush();
         }
         void chooseMRV() {
            Slot best = null;
            for (int i = 0, count, count2 = -1, bestScore = -1, n = TOTAL; i < n; i++) {
               var s = slots[i];
               if (assigned[s.key] != X) continue;
               var pattern = patternForSlot(grid, s);
               var index   = dictIndex[s.length()];
               count = pattern == X ? index.length : candidateCountForPattern(bitset, pattern, index.posBitsets, index.numlong);
               
               if (count == 0) {
                  current = PICK_NOT_DONE;
                  return;
               }
               if (best == null
                   || count < count2
                   || (count == count2 && slotScores[i] > bestScore)) {
                  best      = s;
                  bestScore = slotScores[i];
                  count2    = count;
                  if (count <= 1) break;
               }
            }
            // Re-calculate for the best slot to get actual indices
            if (best == null) {
               current = PICK_DONE;
               return;
            }
            var pattern = patternForSlot(grid, best);
            var index   = dictIndex[best.length()];
            current       = CARRIER;
            current.slot  = best;
            current.count = index.length;
            if (pattern == X) {
               current.indices = null;
               return;
            }
            current.indices = candidateInfoForPattern(bitset, pattern, index.posBitsets, index.numlong);
         }
         boolean backtrack(int depth) {
            if (Thread.currentThread().isInterrupted()) return false;
            nodes++;
            
            if (20_000 > 0 && (System.currentTimeMillis() - t0) > 20_000) return false;
            
            chooseMRV();
            var pick = current;
            if (pick == PICK_DONE) return true;
            if (pick.slot == null) {
               backtracks++;
               return false;
            }
            val info = pick.indices;
            lastMRV = pick.count;
            if (!NO_LOG) renderProgress();
            
            val s     = pick.slot;
            val k     = s.key;
            val entry = dictIndex[s.length()];
            
            if (info != null && info.length > 0) {
               var idxs  = info;
               var L     = idxs.length;
               var tries = Math.min(MAX_TRIES_PER_SLOT, L);
               
               for (var t = 0; t < tries; t++) {
                  var r = rng.nextFloat();
                  //int idxInArray = rng.biasedIndexPow3(L - 1);
                  int idxInArray = (int) (r * r * r * (L - 1));
                  var idx        = idxs[idxInArray];
                  var w          = entry.words[idx];
                  var lemIdx     = Lemma.unpackIndex(w);
                  if (used.get(lemIdx)) continue;
                  
                  if (!placeWord(grid, s, w, undo, depth)) continue;
                  
                  used.set(lemIdx);
                  assigned[k] = w;
                  
                  if (backtrack(depth + 1)) return true;
                  
                  assigned[k] = X;
                  used.clear(lemIdx);
                  grid.undoPlace(undo[depth << 1], undo[(depth << 1) | 1]);
               }
               backtracks++;
               return false;
            }
            
            var N = entry.words.length;
            
            var tries = Math.min(MAX_TRIES_PER_SLOT, N);
            for (var t = 0; t < tries; t++) {
               double r          = rng.nextFloat();
               int    idxInArray = (int) (r * r * r * (N - 1));
               var    w          = entry.words[idxInArray];
               var    lemIdx     = Lemma.unpackIndex(w);
               if (used.get(lemIdx)) continue;
               
               if (!placeWord(grid, s, w, undo, depth)) continue;
               
               used.set(lemIdx);
               assigned[k] = w;
               
               if (backtrack(depth + 1)) return true;
               
               assigned[k] = X;
               used.clear(lemIdx);
               grid.undoPlace(undo[depth << 1], undo[(depth << 1) | 1]);
            }
            
            backtracks++;
            return false;
         }
      }
      
      // initial render (same feel)
      var solver = new Solver();
      if (!NO_LOG) solver.renderProgress();
      var ok = solver.backtrack(0);
      // final progress line
      
      var res = new FillResult(ok, new Gridded(grid), assigned, new FillStats(solver.nodes, solver.backtracks, (System.currentTimeMillis() - t0) / 1000.0, solver.lastMRV));
      if (!multiThreaded) {
         System.out.print("\r" + Strings.padRight("", 120) + "\r");
         System.out.flush();
      }
      // print a final progress line
      if (Main.VERBOSE && !multiThreaded) {
         System.out.println(
                 String.format(Locale.ROOT,
                               "[######################] %d/%d slots | nodes=%d | backtracks=%d | mrv=%d | %.1fs",
                               res.wordCount(), TOTAL, res.nodes(), res.backtracks(), res.lastMRV(), res.seconds()
                              )
                           );
      }
      
      return res;
   }
}