From 944320a2d27a5d2321a2032f2cf76e747b4dbaec Mon Sep 17 00:00:00 2001 From: mike Date: Sun, 28 Dec 2025 09:54:18 +0100 Subject: [PATCH] Gather data --- src/puzzle/DailyGenerator.java | 4 +- src/puzzle/Main.java | 10 +++ src/puzzle/MainTest.java | 32 ++++++++ src/puzzle/SwedishGenerator.java | 103 +++++++++++++++++-------- src/puzzle/ThemePoolBuilderLength.java | 20 ++--- src/test/java/puzzle/MainTest.java | 29 ------- 6 files changed, 125 insertions(+), 73 deletions(-) create mode 100644 src/puzzle/MainTest.java delete mode 100644 src/test/java/puzzle/MainTest.java diff --git a/src/puzzle/DailyGenerator.java b/src/puzzle/DailyGenerator.java index 7244422..ead7727 100644 --- a/src/puzzle/DailyGenerator.java +++ b/src/puzzle/DailyGenerator.java @@ -181,8 +181,8 @@ public class DailyGenerator { var rng = new SwedishGenerator.Rng(opts.seed); for (var attempt = 1; attempt <= opts.tries; attempt++) { - var mask = SwedishGenerator.generateMask(rng, dict.lenCounts, opts.pop, opts.gens); - var filled = SwedishGenerator.fillMask(rng, mask, dict.index, llmScores, 200, 30000); + var mask = SwedishGenerator.generateMask(rng, dict.lenCounts, opts.pop, opts.gens, true); + var filled = SwedishGenerator.fillMask(rng, mask, dict.index, llmScores, 200, 30000, true); if (filled.ok) { return new SwedishGenerator.PuzzleResult(mask, filled); diff --git a/src/puzzle/Main.java b/src/puzzle/Main.java index 30c90af..80d01d7 100644 --- a/src/puzzle/Main.java +++ b/src/puzzle/Main.java @@ -1,5 +1,7 @@ package puzzle; +import puzzle.SwedishGenerator.PuzzleResult; +import puzzle.SwedishGenerator.Rng; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -8,7 +10,15 @@ import java.nio.file.Paths; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Locale; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import static puzzle.SwedishGenerator.fillMask; +import static puzzle.SwedishGenerator.generateMask; +import static puzzle.SwedishGenerator.loadScores; +import static puzzle.SwedishGenerator.loadWords; public class Main { // ---------------- CLI ---------------- diff --git a/src/puzzle/MainTest.java b/src/puzzle/MainTest.java new file mode 100644 index 0000000..d908843 --- /dev/null +++ b/src/puzzle/MainTest.java @@ -0,0 +1,32 @@ +package puzzle; + +//import org.junit.jupiter.api.Test; +//import static org.junit.jupiter.api.Assertions.*; + +public class MainTest { + + static void main() { + new MainTest().testGeneratePuzzle(); + } + // @Test + public void testGeneratePuzzle() { + // Arrange + var opts = new Main.Opts(); + opts.seed = 1234; + opts.pop = 18; + opts.gens = 300; + opts.wordsPath = "src/test/resources/puzzle/pool.txt"; + opts.minSimplicity = 0; + opts.threads = 1; + opts.tries = 1; + + // Act + var result = Main.generatePuzzle(opts); + + // Assert + /* assertNotNull(result); + assertNotNull(result.mask()); + assertNotNull(result.filled()); + assertTrue(result.filled().ok);*/ + } +} diff --git a/src/puzzle/SwedishGenerator.java b/src/puzzle/SwedishGenerator.java index 2f09f90..d761854 100644 --- a/src/puzzle/SwedishGenerator.java +++ b/src/puzzle/SwedishGenerator.java @@ -5,6 +5,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.IntStream; /** @@ -18,7 +20,7 @@ import java.util.stream.IntStream; public class SwedishGenerator { static final int W = 9, H = 8, - CLUE_SIZE = 6, + CLUE_SIZE = 4, SIMPLICITY_DEFAULT_SCORE = 5; static final int MIN_LEN = 2, MAX_LEN = 8; // Directions for '1'..'6' @@ -494,7 +496,7 @@ public class SwedishGenerator { var c = rng.randint(0, W - 1); if (isDigit(g[r][c])) continue; - var d = (char) ('0' + rng.randint(1, r == 0 ? CLUE_SIZE : 4)); + var d = (char) ('0' + rng.randint(1, c == 0 ? CLUE_SIZE : 4)); g[r][c] = d; if (!hasRoomForClue(g, r, c, d)) { g[r][c] = '#'; @@ -519,7 +521,7 @@ public class SwedishGenerator { if (isDigit(cur)) { g[rr][cc] = '#'; } else { - var d = (char) ('0' + rng.randint(1, rr == 0 ? CLUE_SIZE : 4)); + var d = (char) ('0' + rng.randint(1, cc == 0 ? CLUE_SIZE : 4)); g[rr][cc] = d; if (!hasRoomForClue(g, rr, cc, d)) g[rr][cc] = '#'; } @@ -575,8 +577,8 @@ public class SwedishGenerator { return same / (double) (W * H); } - static char[][] generateMask(Rng rng, HashMap lenCounts, int popSize, int gens) { - System.out.println("generateMask init pop: " + popSize); + static char[][] generateMask(Rng rng, HashMap lenCounts, int popSize, int gens, boolean verbose) { + if (verbose) System.out.println("generateMask init pop: " + popSize); var pop = new ArrayList(); for (var i = 0; i < popSize; i++) { @@ -585,6 +587,7 @@ public class SwedishGenerator { } for (var gen = 0; gen < gens; gen++) { + if (Thread.currentThread().isInterrupted()) break; var children = new ArrayList(); var pairs = Math.max(popSize, (int) Math.floor(popSize * 1.5)); @@ -612,7 +615,7 @@ public class SwedishGenerator { } pop = next; - if (gen % 10 == 0) { + if (verbose && gen % 10 == 0) { var bestF = maskFitness(pop.get(0), lenCounts); System.out.println(" gen " + gen + "/" + gens + " bestFitness=" + bestF); } @@ -690,7 +693,7 @@ public class SwedishGenerator { static FillResult fillMask(Rng rng, char[][] mask, HashMap dictIndex, Map llmScores, - int logEveryMs, int timeLimitMs) { + int logEveryMs, int timeLimitMs, boolean verbose) { var grid = deepCopyGrid(mask); var allSlots = extractSlots(grid); @@ -711,6 +714,7 @@ public class SwedishGenerator { final var BAR_LEN = 22; Runnable renderProgress = () -> { + if (!verbose) return; var now = System.currentTimeMillis(); if ((now - lastLog.get()) < logEveryMs) return; lastLog.set(now); @@ -792,6 +796,7 @@ public class SwedishGenerator { class Solver { boolean backtrack() { + if (Thread.currentThread().isInterrupted()) return false; stats.nodes++; if (timeLimitMs > 0 && (System.currentTimeMillis() - t0) > timeLimitMs) return false; @@ -894,12 +899,14 @@ public class SwedishGenerator { } // print a final progress line - System.out.println( - String.format(Locale.ROOT, - "[######################] %d/%d slots | nodes=%d | backtracks=%d | mrv=%d | %.1fs", - assigned.size(), TOTAL, stats.nodes, stats.backtracks, stats.lastMRV, stats.seconds - ) - ); + if (verbose) { + System.out.println( + String.format(Locale.ROOT, + "[######################] %d/%d slots | nodes=%d | backtracks=%d | mrv=%d | %.1fs", + assigned.size(), TOTAL, stats.nodes, stats.backtracks, stats.lastMRV, stats.seconds + ) + ); + } return res; } @@ -913,32 +920,62 @@ public class SwedishGenerator { public record PuzzleResult(char[][] mask, FillResult filled) { } public static PuzzleResult generatePuzzle(Main.Opts opts) { - var rng = new Rng(opts.seed); var llmScores = loadScores(); var tLoad0 = System.nanoTime(); var dict = loadWords(opts.wordsPath, llmScores); var tLoad1 = System.nanoTime(); - System.out.printf(Locale.ROOT, "LOAD_WORDS: %.3fs%n %s words", (tLoad1 - tLoad0) / 1e9, dict.words.size()); + System.out.printf(Locale.ROOT, "LOAD_WORDS: %.3fs%n %s words%n", (tLoad1 - tLoad0) / 1e9, dict.words.size()); - for (var attempt = 1; attempt <= opts.tries; attempt++) { - System.out.println("\nAttempt " + attempt + "/" + opts.tries); - - var tMask0 = System.nanoTime(); - var mask = generateMask(rng, dict.lenCounts, opts.pop, opts.gens); - var tMask1 = System.nanoTime(); - System.out.printf(Locale.ROOT, "MASK: %.3fs%n", (tMask1 - tMask0) / 1e9); - - var tFill0 = System.nanoTime(); - var filled = fillMask(rng, mask, dict.index, llmScores, 200, 60000); - var tFill1 = System.nanoTime(); - System.out.printf(Locale.ROOT, "FILL: %.3fms | Simplicity: %.2f%n", (tFill1 - tFill0) / 1e6, filled.simplicity); - - if (filled.ok && (opts.minSimplicity <= 0 || filled.simplicity >= opts.minSimplicity)) { - return new PuzzleResult(mask, filled); + if (opts.threads > 1) { + System.out.println("Running in multi-threaded mode with " + opts.threads + " threads..."); + var executor = Executors.newFixedThreadPool(opts.threads); + try { + var tasks = new ArrayList>(); + for (int i = 1; i <= opts.tries; i++) { + final int attempt = i; + tasks.add(() -> { + var threadRng = new Rng(opts.seed + attempt); + var mask = generateMask(threadRng, dict.lenCounts, opts.pop, opts.gens, false); + var filled = fillMask(threadRng, mask, dict.index, llmScores, 200, 60000, false); + + if (filled.ok && (opts.minSimplicity <= 0 || filled.simplicity >= opts.minSimplicity)) { + System.out.println("\nSolution found on attempt " + attempt); + return new PuzzleResult(mask, filled); + } + throw new RuntimeException("No solution found in attempt " + attempt); + }); + } + return executor.invokeAny(tasks); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + // all failed + } finally { + executor.shutdownNow(); } - if (filled.ok) { - System.out.printf(Locale.ROOT, "Puzzle simplicity %.2f is below min %.2f, retrying...%n", - filled.simplicity, opts.minSimplicity); + return null; + } else { + var rng = new Rng(opts.seed); + for (var attempt = 1; attempt <= opts.tries; attempt++) { + System.out.println("\nAttempt " + attempt + "/" + opts.tries); + + var tMask0 = System.nanoTime(); + var mask = generateMask(rng, dict.lenCounts, opts.pop, opts.gens, true); + var tMask1 = System.nanoTime(); + System.out.printf(Locale.ROOT, "MASK: %.3fs%n", (tMask1 - tMask0) / 1e9); + + var tFill0 = System.nanoTime(); + var filled = fillMask(rng, mask, dict.index, llmScores, 200, 60000, true); + var tFill1 = System.nanoTime(); + System.out.printf(Locale.ROOT, "FILL: %.3fms | Simplicity: %.2f%n", (tFill1 - tFill0) / 1e6, filled.simplicity); + + if (filled.ok && (opts.minSimplicity <= 0 || filled.simplicity >= opts.minSimplicity)) { + return new PuzzleResult(mask, filled); + } + if (filled.ok) { + System.out.printf(Locale.ROOT, "Puzzle simplicity %.2f is below min %.2f, retrying...%n", + filled.simplicity, opts.minSimplicity); + } } } diff --git a/src/puzzle/ThemePoolBuilderLength.java b/src/puzzle/ThemePoolBuilderLength.java index 8c2ae56..df099ac 100644 --- a/src/puzzle/ThemePoolBuilderLength.java +++ b/src/puzzle/ThemePoolBuilderLength.java @@ -58,18 +58,19 @@ public class ThemePoolBuilderLength { private static final String BROWSER_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"; - static int MIN_SIMPLICITY = 520; + static int MIN_SIMPLICITY = 520, + MAX_WORD_LENGTH = 7; static final class Opts { - String endpoint = "https://jarvis-lan.appmodel.nl/api/stoic/"; + String endpoint = "https://jarvis-lan.appmodel.nl/api/ollama/"; List feeds = new ArrayList<>(DEFAULT_FEEDS); String outDir = "/data/puzzle"; int bridgeN = 30000; int themeN = 800; int relatedN = 2200; int rssItemsPerFeed = 10; - String model = "mistralai/mistral-nemo-instruct-2407"; + String model = "/models/Hadiseh-Mhd/Mixtral-8x7B-Instruct-v0.1-Q4_K_M-GGUF/mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf"; int timeoutSeconds = 180; int retries = 2; int minLen2 = 1000; @@ -78,7 +79,7 @@ public class ThemePoolBuilderLength { int minLen5 = 1000; // set if you also want to force 5-letter words, etc. int minLen6 = 1000; int minLen7 = 1000; - int minLen8 = 1000; + int minLen8 = MAX_WORD_LENGTH >= 8 ? 1000 : 0; } public static void main(String[] args) throws Exception { @@ -94,7 +95,7 @@ public class ThemePoolBuilderLength { lex = loadLexicon(c); } - System.out.println("Master words (2-8, A-Z): " + lex.words.size()); + System.out.println("Master words (2-" + MAX_WORD_LENGTH + ", A-Z): " + lex.words.size()); // RSS via curl (browser-like) var all = new ArrayList(); @@ -118,7 +119,7 @@ public class ThemePoolBuilderLength { var modelId = o.model; if (modelId == null) { var modelsUrl = apiUrl(o.endpoint, "/models"); - System.out.println("LM Studio GET: " + modelsUrl); + System.out.println("Ollama GET: " + modelsUrl); var modelsJson = curlGetJson(o, modelsUrl); modelId = pickModelId(modelsJson); if (modelId == null) { @@ -127,7 +128,7 @@ public class ThemePoolBuilderLength { } System.out.println("Using model: " + modelId); System.out.println("Generating theme words via LM Studio..."); - var llmWords = llmThemeWords(o, modelId, rssText.toString()); + var llmWords = List.of();//llmThemeWords(o, modelId, rssText.toString()); var themeKept = new LinkedHashSet(); for (var wRaw : llmWords) { @@ -398,9 +399,10 @@ public class ThemePoolBuilderLength { final var sql = """ SELECT woord, 10-level_1_to_10, hint - FROM export_words_with_hints_2_8 + FROM export_real_words_with_hints + where length(woord)<=7 order by level_1_to_10 asc - """; + """ ; try (var ps = conn.prepareStatement(sql); var rs = ps.executeQuery()) { diff --git a/src/test/java/puzzle/MainTest.java b/src/test/java/puzzle/MainTest.java deleted file mode 100644 index b7eb2d6..0000000 --- a/src/test/java/puzzle/MainTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package puzzle; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class MainTest { - - @Test - public void testGeneratePuzzle() { - // Arrange - var opts = new Main.Opts(); - opts.seed = 1234; - opts.pop = 18; - opts.gens = 300; - opts.wordsPath = "src/test/resources/puzzle/pool.txt"; - opts.minSimplicity = 0; - opts.threads = 1; - opts.tries = 1; - - // Act - var result = Main.generatePuzzle(opts); - - // Assert - assertNotNull(result); - assertNotNull(result.mask()); - assertNotNull(result.filled()); - assertTrue(result.filled().ok); - } -}