Gather data

src/main/java/puzzle/ExportFormat.java

@@ -26,14 +26,14 @@ public final class ExportFormat {
    
    public static ExportedPuzzle exportFormatFromFilled(PuzzleResult puz, int difficulty, Rewards rewards) {
       Objects.requireNonNull(puz, "puz");
-      var g = puz.filled().grid;
+      var g = puz.filled().grid();
       var H = g.length;
       var W = g[0].length;
       
       // 1) extract "placed" list from all clue digits in the filled grid
       var placed   = new ArrayList<Placed>();
       var allSlots = extractSlots(g);
-      var clueMap  = puz.filled().clueMap;
+      var clueMap  = puz.filled().clueMap();
       
       for (var s : allSlots) {
          var word = clueMap.get(s.key());
@@ -170,7 +170,7 @@ public final class ExportFormat {
       return new Placed(
               lemma,
               lemma.word(),
-              dict.words().get(lemma.word()).clue().toArray(String[]::new), // clue placeholder
+              lemma.clue()/*dict.words()[lemma.index()].clue()*/.toArray(String[]::new), // clue placeholder
               startRow,
               startCol,
               direction,

src/main/java/puzzle/HintScores.java

@@ -1,6 +1,7 @@
 package puzzle;
 
 import java.sql.*;
+import java.util.Map;
 import java.util.function.ToIntFunction;
 
 public final class HintScores {
@@ -8,9 +9,37 @@ public final class HintScores {
    public static void main(String[] args) throws Exception {
       Class.forName("org.sqlite.JDBC");
       try (Connection conn = DriverManager.getConnection("jdbc:sqlite:/home/mike/dev/puzzle-generator/tools/hint/hint.sqlite")) {
-         updateCrossScores(conn, HintScores::exampleScore, 1000);
+         updateCrossScores(conn, HintScores::crossabilityScore, 1000);
       }
    }
+   static final Map<Character, Integer> LETTER_WEIGHT = Map.ofEntries(
+           Map.entry('E', 10), Map.entry('N', 9), Map.entry('A', 9), Map.entry('R', 8),
+           Map.entry('I', 8), Map.entry('O', 7), Map.entry('S', 7), Map.entry('T', 7),
+           Map.entry('D', 6), Map.entry('L', 6), Map.entry('K', 5), Map.entry('M', 5),
+           Map.entry('U', 5), Map.entry('P', 4), Map.entry('G', 4), Map.entry('H', 4),
+           Map.entry('V', 4), Map.entry('B', 3), Map.entry('W', 3),
+           Map.entry('C', 2), Map.entry('F', 2), Map.entry('Z', 2),
+           Map.entry('J', 1), Map.entry('Y', 1), Map.entry('Q', 0), Map.entry('X', 0)
+                                                                     );
+   
+   static boolean isVowel(char ch) {
+      return ch == 'A' || ch == 'E' || ch == 'I' || ch == 'O' || ch == 'U';
+   }
+   
+   static int crossabilityScore(String w) {
+      var score  = 0;
+      var vowels = 0;
+      for (var i = 0; i < w.length(); i++) {
+         var ch = w.charAt(i);
+         score += LETTER_WEIGHT.getOrDefault(ch, 2);
+         if (isVowel(ch)) vowels++;
+      }
+      var ratio = vowels / (double) w.length();
+      if (ratio >= 0.35 && ratio <= 0.65) score += 8;
+      if (w.indexOf('Q') >= 0 || w.indexOf('X') >= 0) score -= 6;
+      if (w.indexOf('Y') >= 0 || w.indexOf('J') >= 0) score -= 2;
+      return score;
+   }
    /**
     * Updates hints.cross_score by computing a score from hints.word.
     *
@@ -83,10 +112,4 @@ public final class HintScores {
          conn.setAutoCommit(prevAutoCommit);
       }
    }
-   
-   // Example scoring callback
-   public static int exampleScore(String word) {
-      return ThemePoolBuilderLength.crossabilityScore(word);
-   }
-   
 }

src/main/java/puzzle/Main.java

@@ -13,8 +13,8 @@ import java.util.*;
 import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
+import static puzzle.SwedishGenerator.*;
 import static puzzle.SwedishGenerator.fillMask;
-import static puzzle.SwedishGenerator.generateMask;
 import static puzzle.SwedishGenerator.loadWords;
 
 public class Main {
@@ -39,6 +39,8 @@ public class Main {
       public int     threads       = Math.max(1, Runtime.getRuntime().availableProcessors());
       public int     tries         = threads;
       public boolean reindex       = false;
+      public int     fillTimeout   = 20_000;
+      public boolean verbose       = false;
    }
    
    public void main(String[] args) {
@@ -68,16 +70,16 @@ public class Main {
       }
       
       section("Result");
-      info(String.format(Locale.ROOT, "simplicity  : %.2f", res.filled().simplicity));
+      info(String.format(Locale.ROOT, "simplicity  : %.2f", res.filled().simplicity()));
       
       section("Mask");
-      System.out.print(indentLines(SwedishGenerator.gridToString(res.mask()), "  "));
+      System.out.print(indentLines(gridToString(res.mask()), "  "));
       
       section("Grid (raw)");
-      System.out.print(indentLines(SwedishGenerator.gridToString(res.filled().grid), "  "));
+      System.out.print(indentLines(gridToString(res.filled().grid()), "  "));
       
       section("Grid (human)");
-      System.out.print(indentLines(SwedishGenerator.renderHuman(res.filled().grid), "  "));
+      System.out.print(indentLines(renderHuman(res.filled().grid()), "  "));
       
       var exported = ExportFormat.exportFormatFromFilled(res, 1, new ExportFormat.Rewards(50, 2, 1));
       
@@ -238,13 +240,13 @@ public class Main {
       var tLoad1 = System.nanoTime();
       
       section("Load");
-      info(String.format(Locale.ROOT, "words        : %,d", dict.words().size()));
+      info(String.format(Locale.ROOT, "words        : %,d", dict.wordz().length));
       info(String.format(Locale.ROOT, "loadTime     : %.3f s", (tLoad1 - tLoad0) / 1e9));
       
       section("Search");
       
       var deadline    = System.currentTimeMillis() + 40_000;
-      var fillTimeout = 20_000;
+     
       
       if (opts.threads > 1) {
          info("mode         : multi-threaded (" + opts.threads + ")");
@@ -256,16 +258,7 @@ public class Main {
             // Keep at least some tasks in flight
             for (int i = 0; i < opts.threads; i++) {
                final int attempt = ++submitted;
-               completionService.submit(() -> {
+               completionService.submit(() -> attempt(new Rng(opts.seed + attempt), dict, opts ));
-                  var threadRng = new Rng(opts.seed + attempt);
-                  var mask      = generateMask(threadRng, dict.lenCounts(), opts.pop, opts.gens, false);
-                  var filled    = fillMask(threadRng, mask, dict.index(), 200, fillTimeout, false);
-                  
-                  if (filled.ok && (opts.minSimplicity <= 0 || filled.simplicity >= opts.minSimplicity)) {
-                     return new PuzzleResult(dict, mask, filled);
-                  }
-                  return null;
-               });
             }
             
             while (System.currentTimeMillis() < deadline) {
@@ -281,16 +274,7 @@ public class Main {
                // Submit another task if we still have time
                if (System.currentTimeMillis() < deadline) {
                   final int attempt = ++submitted;
-                  completionService.submit(() -> {
+                  completionService.submit(() -> attempt(new Rng(opts.seed + attempt), dict, opts  ));
-                     var threadRng = new Rng(opts.seed + attempt);
-                     var mask      = generateMask(threadRng, dict.lenCounts(), opts.pop, opts.gens, false);
-                     var filled    = fillMask(threadRng, mask, dict.index(), 200, fillTimeout, false);
-                     
-                     if (filled.ok && (opts.minSimplicity <= 0 || filled.simplicity >= opts.minSimplicity)) {
-                        return new PuzzleResult(dict, mask, filled);
-                     }
-                     return null;
-                  });
                }
             }
             warn("status       : UNSOLVED (timeout)");
@@ -313,20 +297,11 @@ public class Main {
             attempt++;
             info("try          : " + attempt + " (remaining: " + (deadline - System.currentTimeMillis()) / 1000 + "s)");
             
-            var mask   = generateMask(rng, dict.lenCounts(), opts.pop, opts.gens, true);
+            var result = attempt(rng, dict, opts);
-            var filled = fillMask(rng, mask, dict.index(), 200, fillTimeout, true);
+            if (result != null) {
-            
-            if (filled.ok && (opts.minSimplicity <= 0 || filled.simplicity >= opts.minSimplicity)) {
                info("status       : SOLVED");
                info("foundAtTry   : " + attempt);
-               return new PuzzleResult(dict, mask, filled);
+               return result;
-            }
-            
-            if (filled.ok) {
-               warn(String.format(Locale.ROOT,
-                                  "simplicity   : %.2f (below min %.2f)",
-                                  filled.simplicity, opts.minSimplicity
-                                 ));
             }
          }
          
@@ -335,6 +310,23 @@ public class Main {
       }
    }
    
+   static PuzzleResult attempt(Rng rng, Dict dict, Opts opts ) {
+      var mask   = SwedishGenerator.generateMask(rng, dict.lenCounts(), opts.pop, opts.gens, opts.verbose);
+      var filled = fillMask(rng, mask, dict.index(), 200, opts.fillTimeout, opts.verbose);
+      
+      if (filled.ok() && (opts.minSimplicity <= 0 || filled.simplicity() >= opts.minSimplicity)) {
+         return new PuzzleResult(dict, mask, filled);
+      }
+      
+      if (opts.verbose && filled.ok()) {
+         System.err.printf(Locale.ROOT,
+                           "simplicity   : %.2f (below min %.2f)%n",
+                           filled.simplicity(), opts.minSimplicity
+                          );
+      }
+      return null;
+   }
+   
    // ---------------- Export (unchanged logic) ----------------
    
    private static String toJson(ExportFormat.ExportedPuzzle puzzle, String date, String theme) {

src/main/java/puzzle/SwedishGenerator.java

@@ -6,7 +6,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 /**
  * SwedishGenerator.java
@@ -19,6 +18,7 @@ import java.util.stream.Collectors;
 public class SwedishGenerator {
    
    static final int W               = 9, H = 8,
+           SIZE                     = W * H,
            CLUE_SIZE                = 4,
            SIMPLICITY_DEFAULT_SCORE = 2;
    static final int MIN_LEN = 2, MAX_LEN = 8;
@@ -138,15 +138,14 @@ public class SwedishGenerator {
       }
    }
    
-   static record Lemma(int index, String word, int length, int difficulty, int simpel, int score, int cross, ArrayList<String> clue) {
+   static record Lemma(int index, String word, int length, int difficulty, int simpel, int score, ArrayList<String> clue) {
       
       static int LEMMA_COUNTER = 0;
-      public Lemma(String word, int simpel, int score, String clue) {
+      public Lemma(int index, String word, int simpel, int score, String clue) {
          var complex = 0 + ((8 - word.length()) * 30) + ((10 - score) * 15);
-         var crossScore = ThemePoolBuilderLength.crossabilityScore(word);
          var list    = new ArrayList<String>(10);
          list.add(clue);
-         this(++LEMMA_COUNTER, word, word.length(), complex, simpel, score, (crossScore * 7) + ((score) * 30) + ((word.length()) * 15), list);
+         this(index, word, word.length(), complex, simpel, score, list);
          
          // Prioritize simple words (high lScore) and long words.
          // lScore (1-10) adds up to 1000 points (weight 100).
@@ -158,6 +157,7 @@ public class SwedishGenerator {
          // Length impact: up to 8 * 10 = 80
          // Score impact: up to 10 * 15 = 150
       }
+      public Lemma(String word, int simpel, int score, String clue) { this(LEMMA_COUNTER++, word, simpel, score, clue); }
       char charAt(int idx)                                          { return word.charAt(idx); }
       @Override public int hashCode()                               { return index; }
       @Override public boolean equals(Object o) {
@@ -166,9 +166,37 @@ public class SwedishGenerator {
       }
    }
    
-   public static record Dict(Map<String, Lemma> words,
+   public static record Dict(Lemma[] wordz,
                              DictEntry[] index,
-                             int[] lenCounts) { }
+                             int[] lenCounts) {
+      
+      public Dict(Lemma[] wordz) {
+         // Sort words by difficulty in ascending order
+         Lemma[] words = wordz.clone();
+         Arrays.sort(words, Comparator.comparingInt(wd -> wd.simpel));
+         
+         var lenCounts = new int[12];
+         var index     = new DictEntry[12];
+         Arrays.setAll(index, i -> new DictEntry(i));
+         int maxLength = -1;
+         for (var w : words) {
+            var L = w.length();
+            if (L > maxLength) maxLength = L;
+            lenCounts[L]++;
+            
+            var entry = index[L];
+            var idx   = entry.words.size();
+            entry.words.add(w);
+            
+            for (var i = 0; i < L; i++) {
+               var letter = w.charAt(i) - 'A';
+               if (letter >= 0 && letter < 26) entry.pos[i][letter].add(idx);
+               else throw new RuntimeException("Illegal letter: " + letter + " in word " + w);
+            }
+         }
+         this(wordz, index, lenCounts);
+      }
+   }
    static Dict loadWords(String wordsPath) {
       String raw;
       try {
@@ -191,11 +219,10 @@ public class SwedishGenerator {
          first = false;
          var s = word.toUpperCase(Locale.ROOT);
          if (s.matches("^[A-Z]{2,8}$")) {
-            int score  = SIMPLICITY_DEFAULT_SCORE;
+            
-            int simpel = 0;
             // CSV has level 1-10. llmScores use 10-level.
-            score  = 10 - Integer.parseInt(parts[1].trim());
+            int score   = 10 - Integer.parseInt(parts[1].trim());
-            simpel = Integer.parseInt(parts[2].trim());
+            int simpel  = Integer.parseInt(parts[2].trim());
             var rawClue = parts[3].trim();
             if (rawClue.startsWith("\"") && rawClue.endsWith("\"")) {
                rawClue = rawClue.substring(1, rawClue.length() - 1).replace("\"\"", "\"");
@@ -207,33 +234,12 @@ public class SwedishGenerator {
                   map.put(s, new Lemma(s, simpel, score, rawClue));
                }
             }
-         }
+         } else {
-      }
+            System.err.println("Invalid word: " + line);
-      var words = map.values().stream().collect(Collectors.toCollection(ArrayList::new));
-      // Sort words by difficulty in ascending order
-      words.sort(Comparator.comparingInt(wd -> wd.simpel));
-      
-      var lenCounts = new int[12];
-      var index     = new DictEntry[12];
-      Arrays.setAll(index, i -> new DictEntry(i));
-      int maxLength = -1;
-      for (var w : words) {
-         var L = w.length();
-         if (L > maxLength) maxLength = L;
-         lenCounts[L]++;
-         
-         var entry = index[L];
-         var idx   = entry.words.size();
-         entry.words.add(w);
-         
-         for (var i = 0; i < L; i++) {
-            var letter = w.charAt(i) - 'A';
-            if (letter >= 0 && letter < 26) entry.pos[i][letter].add(idx);
-            else throw new RuntimeException("Illegal letter: " + letter + " in word " + w);
          }
       }
       
-      return new Dict(map, index, lenCounts);
+      return new Dict(map.values().toArray(Lemma[]::new));
    }
    
    static int[] intersectSorted(int[] a, int aLen, int[] b, int bLen) {
@@ -284,13 +290,9 @@ public class SwedishGenerator {
    }
    // ---------------- Slots ----------------
    
-   static record Slot(int clueR, int clueC, char dir, int[] rs, int[] cs, int len) {
+   static record Slot(String key, int clueR, int clueC, char dir, int[] rs, int[] cs, int len) {
       
-      public Slot(int clueR, int clueC, char dir, int[] rs, int[] cs) {
+      public Slot(int clueR, int clueC, char dir, int[] rs, int[] cs) { this(clueR + "," + clueC + ":" + dir, clueR, clueC, dir, rs, cs, rs.length); }
-         this(clueR, clueC, dir, rs, cs, rs.length);
-         
-      }
-      String key() { return clueR + "," + clueC + ":" + dir; }
    }
    
    static ArrayList<Slot> extractSlots(char[][] grid) {
@@ -345,14 +347,19 @@ public class SwedishGenerator {
    }
    
    // ---------------- FAST mask fitness ----------------
-   
+   final static int[][] nbrs8 = new int[][]{
+           { -1, -1 }, { -1, 0 }, { -1, 1 },
+           { 0, -1 }, { 0, 1 },
+           { 1, -1 }, { 1, 0 }, { 1, 1 }
+   };
+   static final int[][] nbrs4 = new int[][]{ { -1, 0 }, { 1, 0 }, { 0, -1 }, { 0, 1 } };
    static long maskFitness(char[][] grid, int[] lenCounts) {
       long penalty = 0;
       
       var clueCount = 0;
       for (var r = 0; r < H; r++) for (var c = 0; c < W; c++) if (isDigit(grid[r][c])) clueCount++;
       
-      var targetClues = (int) Math.round(W * H * 0.25); // ~18
+      var targetClues = (int) Math.round(SIZE * 0.25); // ~18
       penalty += 8L * Math.abs(clueCount - targetClues);
       
       var slots = extractSlots(grid);
@@ -368,7 +375,7 @@ public class SwedishGenerator {
          if (s.len > MAX_LEN) penalty += 8000 + (long) (s.len - MAX_LEN) * 500L;
          
          if (s.len >= MIN_LEN && s.len <= MAX_LEN) {
-            if (lenCounts[s.len]<=0) penalty += 12000;
+            if (lenCounts[s.len] <= 0) penalty += 12000;
          }
          
          for (var i = 0; i < s.len; i++) {
@@ -389,13 +396,8 @@ public class SwedishGenerator {
       
       // clue clustering (8-connected)
       var seen  = new boolean[H][W];
-      var stack = new int[W * H];
+      var stack = new int[SIZE];
       int sp;
-      var nbrs8 = new int[][]{
-              { -1, -1 }, { -1, 0 }, { -1, 1 },
-              { 0, -1 }, { 0, 1 },
-              { 1, -1 }, { 1, 0 }, { 1, 1 }
-      };
       
       for (var r = 0; r < H; r++)
          for (var c = 0; c < W; c++) {
@@ -424,7 +426,7 @@ public class SwedishGenerator {
          }
       
       // dead-end-ish letter cell (3+ walls)
-      var nbrs4 = new int[][]{ { -1, 0 }, { 1, 0 }, { 0, -1 }, { 0, 1 } };
+      
       for (var r = 0; r < H; r++)
          for (var c = 0; c < W; c++) {
             if (!isLetterCell(grid[r][c])) continue;
@@ -447,7 +449,7 @@ public class SwedishGenerator {
    
    static char[][] randomMask(Rng rng) {
       var g           = makeEmptyGrid();
-      var targetClues = (int) Math.round(W * H * 0.25);
+      var targetClues = (int) Math.round(SIZE * 0.25);
       int placed      = 0, guard = 0;
       
       while (placed < targetClues && guard++ < 4000) {
@@ -594,17 +596,23 @@ public class SwedishGenerator {
       public int    lastMRV;
    }
    
-   public static final class FillResult {
+   public static final record FillResult(boolean ok,
+                                         char[][] grid,
+                                         HashMap<String, Lemma> clueMap,
+                                         FillStats stats,
+                                         double simplicity) {
       
-      public boolean                ok;
+      public FillResult(boolean ok, char[][] grid, HashMap<String, Lemma> assigned, FillStats stats) {
-      public char[][]               grid;
+         double totalSimplicity = 0;
-      public HashMap<String, Lemma> clueMap;
+         if (ok) {
-      public FillStats              stats;
+            for (var w : assigned.values()) totalSimplicity += w.difficulty;
-      public double                 simplicity;
+            totalSimplicity = assigned.isEmpty() ? 0 : totalSimplicity / assigned.size();
+         }
+         this(ok, grid, assigned, stats, totalSimplicity);
+      }
    }
    
-   record Undo(int[] rs, int[] cs, char[] prev, int n) {
+   record Undo(int[] rs, int[] cs, char[] prev, int n) { }
-   }
    
    static char[] patternForSlot(char[][] grid, Slot s) {
       var pat = new char[s.len];
@@ -645,11 +653,13 @@ public class SwedishGenerator {
       }
       return new Undo(urs, ucs, up, n);
    }
-   
+   static final int MAX_TRIES_PER_SLOT = 2000;
    static void undoPlace(char[][] grid, Undo u) {
       for (var i = 0; i < u.n; i++) grid[u.rs[i]][u.cs[i]] = u.prev[i];
    }
-   
+   record Pick(Slot slot,
+               CandidateInfo info,
+               boolean done) { }
    static FillResult fillMask(Rng rng, char[][] mask, DictEntry[] dictIndex,
                               int logEveryMs, int timeLimitMs, boolean verbose) {
       
@@ -692,12 +702,6 @@ public class SwedishGenerator {
          System.out.flush();
       };
       
-      record Pick(Slot slot,
-                  CandidateInfo info,
-                  boolean done) {
-         
-      }
-      
       java.util.function.Supplier<Pick> chooseMRV = () -> {
          Slot          best     = null;
          CandidateInfo bestInfo = null;
@@ -734,7 +738,6 @@ public class SwedishGenerator {
          }
       };
       IO.println("hit");
-      final var MAX_TRIES_PER_SLOT = 2000;
       
       class Solver {
          
@@ -829,20 +832,8 @@ public class SwedishGenerator {
       System.out.print("\r" + padRight("", 120) + "\r");
       System.out.flush();
       
-      var res = new FillResult();
-      res.ok        = ok;
-      res.grid      = grid;
-      res.clueMap   = assigned;
       stats.seconds = (System.currentTimeMillis() - t0) / 1000.0;
-      res.stats     = stats;
+      var res = new FillResult(ok, grid, assigned, stats);
-      
-      if (ok) {
-         double totalSimplicity = 0;
-         for (var w : assigned.values()) {
-            totalSimplicity += w.difficulty;
-         }
-         res.simplicity = assigned.isEmpty() ? 0 : totalSimplicity / assigned.size();
-      }
       
       // print a final progress line
       if (verbose) {

src/main/java/puzzle/ThemePoolBuilderLength.java

@@ -337,34 +337,7 @@ public class ThemePoolBuilderLength {
       return x;
    }
    
-   static final Map<Character, Integer> LETTER_WEIGHT = Map.ofEntries(
-           Map.entry('E', 10), Map.entry('N', 9), Map.entry('A', 9), Map.entry('R', 8),
-           Map.entry('I', 8), Map.entry('O', 7), Map.entry('S', 7), Map.entry('T', 7),
-           Map.entry('D', 6), Map.entry('L', 6), Map.entry('K', 5), Map.entry('M', 5),
-           Map.entry('U', 5), Map.entry('P', 4), Map.entry('G', 4), Map.entry('H', 4),
-           Map.entry('V', 4), Map.entry('B', 3), Map.entry('W', 3),
-           Map.entry('C', 2), Map.entry('F', 2), Map.entry('Z', 2),
-           Map.entry('J', 1), Map.entry('Y', 1), Map.entry('Q', 0), Map.entry('X', 0)
-                                                                     );
 
-   static boolean isVowel(char ch) {
-      return ch == 'A' || ch == 'E' || ch == 'I' || ch == 'O' || ch == 'U';
-   }
-   
-   static int crossabilityScore(String w) {
-      var score  = 0;
-      var vowels = 0;
-      for (var i = 0; i < w.length(); i++) {
-         var ch = w.charAt(i);
-         score += LETTER_WEIGHT.getOrDefault(ch, 2);
-         if (isVowel(ch)) vowels++;
-      }
-      var ratio = vowels / (double) w.length();
-      if (ratio >= 0.35 && ratio <= 0.65) score += 8;
-      if (w.indexOf('Q') >= 0 || w.indexOf('X') >= 0) score -= 6;
-      if (w.indexOf('Y') >= 0 || w.indexOf('J') >= 0) score -= 2;
-      return score;
-   }
    
    /**
     * @param words  id -> word
@@ -428,7 +401,7 @@ public class ThemePoolBuilderLength {
       
       for (var i = 0; i < n; i++) {
          var w          = out.get(i);
-         var crossScore = crossabilityScore(w);
+         var crossScore = HintScores.crossabilityScore(w);
          var lScore     = levelOf.getOrDefault(w, 5);
          
          // Prioritize simple words (high lScore) and long words.