introduce bitloops

This commit is contained in:
mike
2026-01-17 13:22:04 +01:00
parent 0c56fafeaa
commit 9102dcb922
10 changed files with 706 additions and 82 deletions

View File

@@ -1,252 +0,0 @@
package puzzle;
import com.google.gson.Gson;
import puzzle.SwedishGenerator.Lemma;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.Arrays;
import java.util.function.LongConsumer;
import static java.nio.charset.StandardCharsets.US_ASCII;
public final class CsvIndexService
implements Closeable {
static final ScopedValue<CsvIndexService> SC = ScopedValue.newInstance();
static final Gson GSON = new Gson();
private static final int MAGIC = 0x4C494458; // "LIDX"
private static final int VERSION = 1;
static int SIMPEL_IDX = 3;
private final Path csvPath;
private final Path idxPath;
private volatile long[] offsets; // lazy
private volatile FileChannel csvChannel; // open once
private final Object lock = new Object();
public CsvIndexService(Path csvPath, Path idxPath) {
this.csvPath = csvPath;
this.idxPath = idxPath;
}
public static int lineToSimpel(String line) {
var parts = line.split(",", 5);
return Integer.parseInt(parts[SIMPEL_IDX].trim());
}
public static String[] lineToClue(String line) {
if (line.isBlank()) throw new RuntimeException("Empty line");
var parts = line.split(",", 5);
var rawClue = parts[4].trim();
if (rawClue.startsWith("\"") && rawClue.endsWith("\"")) {
rawClue = rawClue.substring(1, rawClue.length() - 1).replace("\"\"", "\"");
}
return GSON.fromJson(rawClue, String[].class);
}
public static void lineToLemma(String line, LongConsumer ok) {
if (line.isBlank()) {
throw new RuntimeException("Empty line");
}
var parts = line.split(",", 5);
var id = Integer.parseInt(parts[0].trim());
var word = parts[1].trim();
/* if (!word.matches("^[A-Z]{2,8}$")) {
throw new RuntimeException("Invalid word:" + line);
}*/
int score = Integer.parseInt(parts[2].trim());
if (score < 1) {
if (Main.VERBOSE) System.err.println("Word too complex: " + line);
return;
}
ok.accept(Lemma.pack(id, word.getBytes(US_ASCII)));
}
public static int simpel(int index) {
try {
if (SC.isBound())
return lineToSimpel(SC.get().getLine(index));
return -1;
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Failed to get clues for index " + index, e);
}
}
public static String[] clues(int index) {
try {
if (SC.isBound())
return lineToClue(SC.get().getLine(index));
return new String[0];
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Failed to get clues for index " + index, e);
}
}
/** Haal één regel op (0-based line index), met self-healing index (1x rebuild). */
public String getLine(int lineIndex) throws IOException {
ensureLoaded();
var line = readLineAt(lineIndex);
if (startsWithIndex(line, lineIndex)) return line;
// mismatch => rebuild index en nog 1x proberen
synchronized (lock) {
rebuildIndexLocked();
line = readLineAt(lineIndex);
if (startsWithIndex(line, lineIndex)) return line;
}
throw new RuntimeException("Index mismatch after rebuild. Requested=" + lineIndex + ", got line=" + preview(line));
}
public void ensureLoaded() throws IOException {
if (offsets != null && csvChannel != null && csvChannel.isOpen()) return;
synchronized (lock) {
if (offsets != null && csvChannel != null && csvChannel.isOpen()) return;
csvChannel = FileChannel.open(csvPath, StandardOpenOption.READ);
if (Files.exists(idxPath)) {
try {
offsets = readIndex(idxPath);
return;
} catch (IOException badIndex) {
// fall-through -> rebuild
}
}
rebuildIndexLocked();
}
}
private void rebuildIndexLocked() throws IOException {
var built = buildOffsets(csvPath);
writeIndex(idxPath, built);
offsets = built;
}
private String readLineAt(int lineIndex) throws IOException {
var local = offsets;
if (lineIndex < 0 || lineIndex >= local.length) {
throw new IndexOutOfBoundsException("lineIndex=" + lineIndex + ", max=" + (local.length - 1));
}
long currentPos = local[lineIndex];
// lees in blokjes (sneller dan 1 byte) tot newline
var buf = new byte[8192];
var total = 0;
var out = new byte[256];
while (true) {
var bb = ByteBuffer.wrap(buf);
var n = csvChannel.read(bb, currentPos);
if (n < 0) break; // EOF
currentPos += n;
var end = n;
for (var i = 0; i < end; i++) {
var b = buf[i];
if (b == (byte) '\n') {
return new String(out, 0, total, StandardCharsets.UTF_8);
}
if (b == (byte) '\r') continue;
if (total == out.length) out = Arrays.copyOf(out, out.length * 2);
out[total++] = b;
}
}
return new String(out, 0, total, StandardCharsets.UTF_8);
}
/** Check: begint de regel met "<lineIndex>," */
private static boolean startsWithIndex(String line, int lineIndex) {
if (line == null || line.isEmpty()) return false;
var comma = line.indexOf(',');
if (comma <= 0) return false;
// snelle parse zonder split
long v = 0;
for (var i = 0; i < comma; i++) {
var c = line.charAt(i);
if (c < '0' || c > '9') return false;
v = (v * 10) + (c - '0');
if (v > Integer.MAX_VALUE) return false;
}
return v == lineIndex;
}
private static String preview(String s) {
if (s == null) return "null";
return s.length() <= 120 ? s : s.substring(0, 120) + "...";
}
/** Bouw offsets door newlines te scannen. Resultaat is exact getrimd. */
public static long[] buildOffsets(Path path) throws IOException {
try (var ch = FileChannel.open(path, StandardOpenOption.READ)) {
var offs = new long[131072]; // start-capacity, groeit indien nodig
var c = 0;
offs[c++] = 0;
var buf = ByteBuffer.allocateDirect(1 << 20);
int pos = 0;
while (true) {
buf.clear();
var n = ch.read(buf);
if (n < 0) break;
buf.flip();
for (var i = 0; i < n; i++) {
if (buf.get(i) == (byte) '\n') {
if (c == offs.length) offs = Arrays.copyOf(offs, offs.length * 2);
offs[c++] = pos + i + 1;
}
}
pos += n;
}
return Arrays.copyOf(offs, c);
}
}
public static void writeIndex(Path out, long[] offsets) throws IOException {
try (var dos = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(
out, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)))) {
dos.writeInt(MAGIC);
dos.writeInt(VERSION);
dos.writeInt(offsets.length);
for (var v : offsets) dos.writeLong(v);
}
}
public static long[] readIndex(Path in) throws IOException {
try (var dis = new DataInputStream(new BufferedInputStream(Files.newInputStream(in)))) {
if (dis.readInt() != MAGIC) throw new IOException("Not a LIDX file");
var version = dis.readInt();
if (version != VERSION) throw new IOException("Unsupported version: " + version);
var n = dis.readInt();
if (n < 0) throw new IOException("Corrupt length: " + n);
var offsets = new long[n];
for (var i = 0; i < n; i++) offsets[i] = dis.readLong();
return offsets;
}
}
@Override
public void close() throws IOException {
synchronized (lock) {
if (csvChannel != null) csvChannel.close();
csvChannel = null;
offsets = null;
}
}
}

View File

@@ -12,10 +12,6 @@ import puzzle.SwedishGenerator.DictEntry;
import puzzle.SwedishGenerator.FillResult;
import puzzle.SwedishGenerator.Grid;
import puzzle.SwedishGenerator.Slotinfo;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -197,7 +193,7 @@ public record Export() {
}
}
record Placed(long lemma, int slotKey, int[] cells) {
record Placed(long lemma, int shardIdx, int slotKey, int[] cells) {
static final char[] DIRECTION = { Placed.VERTICAL, Placed.HORIZONTAL, Placed.VERTICAL, Placed.HORIZONTAL };
public static final char HORIZONTAL = 'h';
@@ -214,9 +210,10 @@ public record Export() {
public record WordOut(String word, int[] cell, int startRow, int startCol, char direction, int arrowRow, int arrowCol, boolean isReversed, int complex, String[] clue) {
public WordOut(long l, int startRow, int startCol, char d, int arrowRow, int arrowCol, boolean isReversed) {
public WordOut(long l, int shardIdx, int startRow, int startCol, char d, int arrowRow, int arrowCol, boolean isReversed) {
val meta = Meta.readRecord(Meta.shardKey(l), shardIdx);
this(Lemma.asWord(l), new int[]{ arrowRow, arrowCol, startRow, startCol }, startRow, startCol, d, arrowRow, arrowCol, isReversed,
CsvIndexService.simpel(Lemma.unpackIndex(l)), CsvIndexService.clues(Lemma.unpackIndex(l)));
meta.simpel(), meta.clues());
}
}
@@ -230,7 +227,7 @@ public record Export() {
for (var n = 1; n < slots.length; n++) {
if (slots[n].assign().w != X) {
k++;
simpel += CsvIndexService.simpel(Lemma.unpackIndex(slots[n].assign().w));
simpel += Meta.readRecord(Meta.shardKey(slots[n].assign().w), slots[n].assign().shardIdx).simpel();//.simpel(Lemma.unpackIndex(slots[n].assign().w));
}
}
simpel = k == 0 ? 0 : simpel / k;
@@ -239,7 +236,7 @@ public record Export() {
public ExportedPuzzle exportFormatFromFilled(int difficulty, Rewards rewards) {
var placed = new ArrayList<Placed>();
for (var slot : slots) {
placed.add(new Placed(slot.assign().w, slot.key(), Gridded.walk((byte) slot.key(), slot.lo(), slot.hi()).toArray()));
placed.add(new Placed(slot.assign().w, slot.assign().shardIdx, slot.key(), Gridded.walk((byte) slot.key(), slot.lo(), slot.hi()).toArray()));
}
// If nothing placed: return full grid mapped to letters/# only
@@ -285,6 +282,7 @@ public record Export() {
int MIN_R = minR, MIN_C = minC;
var wordsOut = placed.stream().map(p -> new WordOut(
p.lemma,
p.shardIdx,
p.startRow() - MIN_R,
p.startCol() - MIN_C,
p.direction(),
@@ -298,19 +296,9 @@ public record Export() {
interface Dicts {
static Dict loadDict(String wordsPath) {
try {
var map = new LongArrayList(100_000);
Files.lines(Path.of(wordsPath), StandardCharsets.UTF_8).forEach(line -> CsvIndexService.lineToLemma(line, map::add));
return makeDict(map.toArray());
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException("Failed to load dictionary from " + wordsPath, e);
}
}
static Dict makeDict(long[] wordz) {
var index = new DictEntryDTO[SwedishGenerator.MAX_WORD_LENGTH_PLUS_ONE];
Arrays.setAll(index, i -> new DictEntryDTO(i));
Arrays.setAll(index, DictEntryDTO::new);
for (var lemma : wordz) {
var L = Lemma.length(lemma);

View File

@@ -4,6 +4,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.val;
import puzzle.Masker.Clues;
import puzzle.SwedishGenerator.Rng;
import java.io.IOException;
@@ -15,11 +16,10 @@ import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import static puzzle.CsvIndexService.SC;
import static puzzle.Export.*;
import static puzzle.SwedishGenerator.*;
import static puzzle.Export.Dicts.loadDict;
public class Main {
@@ -44,11 +44,12 @@ public class Main {
@NoArgsConstructor
public static class Opts {
static int SSIZE = 20;
public int seed = (int) (System.nanoTime() ^ System.currentTimeMillis());
public int clueSize = 20;
public int pop = 40;
public int offspring = 60;
public int gens = 500;
public int clueSize = SSIZE;
public int pop = SSIZE * 2;
public int offspring = SSIZE * 3;
public int gens = 600;
public String wordsPath = "nl_score_hints_v3.csv";
public double minSimplicity = 0; // 0 means no limit
public int threads = Math.max(1, Runtime.getRuntime().availableProcessors());
@@ -59,16 +60,7 @@ public class Main {
}
void main(String[] args) {
var csv = Paths.get("nl_score_hints_v3.csv");
var idx = Paths.get("nl_score_hints_v3.idx");
try {
val scv = new CsvIndexService(csv, idx);
scv.ensureLoaded();
ScopedValue.where(SC, scv).run(() -> _main(args));
} catch (IOException e) {
throw new RuntimeException(e);
}
_main(args);
}
public void _main(String[] args) {
var opts = parseArgs(args);
@@ -274,7 +266,7 @@ public class Main {
PuzzleResult generatePuzzle(Opts opts) {
var tLoad0 = System.nanoTime();
var dict = loadDict(opts.wordsPath);
var dict = DictData.DICT;//loadDict(opts.wordsPath);
var tLoad1 = System.nanoTime();
section("Load");
@@ -294,10 +286,9 @@ public class Main {
try {
// Keep at least some tasks in flight
final var service = CsvIndexService.SC.get();
for (int i = 0; i < opts.threads; i++) {
final int attemptIdx = ++submitted;
completionService.submit(() -> ScopedValue.where(CsvIndexService.SC, service).call(() -> attempt(new Rng(opts.seed + attemptIdx), dict, opts)));
completionService.submit(() -> attempt(new Rng(opts.seed + attemptIdx), dict, opts));
}
while (System.currentTimeMillis() < deadline) {
@@ -314,7 +305,7 @@ public class Main {
// Submit another task if we still have time
if (System.currentTimeMillis() < deadline) {
final int attemptIdx = ++submitted;
completionService.submit(() -> ScopedValue.where(CsvIndexService.SC, service).call(() -> attempt(new Rng(opts.seed + attemptIdx), dict, opts)));
completionService.submit(() -> attempt(new Rng(opts.seed + attemptIdx), dict, opts));
}
}
if (resFinal == null) warn("status : UNSOLVED (timeout)");
@@ -381,11 +372,44 @@ public class Main {
return null;
}
}
static Clues generateClues() {
String simple = "000 3000\n" +
" 3 \n" +
" 31 \n" +
" 3\n" +
"1 \n" +
"1 \n" +
"1 2\n" +
"1 222 3";
String sampleComplex = "1 0000\n" +
"1 \n" +
"00 01 \n" +
" 1 \n" +
" 1 \n" +
" 2 1 \n" +
" 1 \n" +
"221 22\n";
String def = " 30000\n" +
"0 001 \n" +
" 1 \n" +
" 3 \n" +
" 3 \n" +
" 32 \n" +
" 32 2\n" +
"2222 3";
return Clues.parse(sampleComplex
);
}
static Clues generateNewClues(Rng rng, Opts opts) {
var masker = new Masker(rng, new int[STACK_SIZE], Masker.Clues.createEmpty());
var mask = masker.generateMask(opts.clueSize, opts.pop, opts.gens, opts.offspring);
return mask;
}
static PuzzleResult _attempt(Rng rng, Dict dict, Opts opts) {
long t0 = System.currentTimeMillis();
TOTAL_ATTEMPTS.incrementAndGet();
var masker = new Masker(rng, new int[STACK_SIZE], Masker.Clues.createEmpty());
var mask = masker.generateMask(opts.clueSize, opts.pop, opts.gens, opts.offspring);
val mask = generateNewClues(rng, opts);
//val mask = generateClues();
if (mask == null) return null;
val multiThreaded = Thread.currentThread().getName().contains("pool");
var slots = Masker.extractSlots(mask, dict.index());
@@ -423,7 +447,9 @@ public class Main {
"[ATTEMPT] thread=%s | status=%s | nodes=%d | backtracks=%d | nps=%d | simplicity=%s | time=%.1fs%n",
name, status, filled.nodes(), filled.backtracks(), nps, simplicity, totalTime
);
if (!filled.ok()) {
System.out.println(Arrays.stream(new Clued(mask).gridToString().split("\n")).map(s -> "\"" + s + "\\n\" +").collect(Collectors.joining("\n")));
}
if (filled.ok() && (opts.minSimplicity <= 0 || filled.stats().simplicity >= opts.minSimplicity)) {
return new PuzzleResult(new Clued(mask), new Gridded(grid), slotInfo, filled);
}
@@ -441,7 +467,7 @@ public class Main {
record JsonExportedPuzzle(String date, String theme, int difficulty, Rewards rewards, String[] grid, WordOut[] words) { }
private static String toJson(ExportedPuzzle puzzle, String date, String theme) {
return CsvIndexService.GSON.toJson(new JsonExportedPuzzle(date, theme, puzzle.difficulty(), puzzle.rewards(), puzzle.grid(), puzzle.words()));
return Meta.GSON.toJson(new JsonExportedPuzzle(date, theme, puzzle.difficulty(), puzzle.rewards(), puzzle.grid(), puzzle.words()));
}
private static String escapeJson(String s) {

View File

@@ -436,6 +436,23 @@ public record Masker(Rng rng, int[] stack, Clues cache) {
public long rhi() { return rhi; }
public static Clues createEmpty() { return new Clues(0, 0, 0, 0, 0, 0); }
public static Clues parse(String s) {
var c = createEmpty();
var lines = s.split("\n");
for (int r = 0; r < Math.min(lines.length, R); r++) {
var line = lines[r];
for (int col = 0; col < Math.min(line.length(), C); col++) {
char ch = line.charAt(col);
if (ch >= '0' && ch <= '3') {
int idx = Grid.offset(r, col);
byte dir = (byte) (ch - '0');
if ((idx & 64) == 0) c.setClueLo(1L << idx, dir);
else c.setClueHi(1L << (idx & 63), dir);
}
}
}
return c;
}
public boolean cluelessLo(int idx) {
if (!isClueLo(idx)) return false;
clearClueLo(~(1L << idx));
@@ -506,7 +523,7 @@ public record Masker(Rng rng, int[] stack, Clues cache) {
}
}
static record Slot(int key, long lo, long hi, DictEntry entry) {
public record Slot(int key, long lo, long hi, DictEntry entry) {
static final int BIT_FOR_DIR = 2;
static Slot from(int key, long lo, long hi, DictEntry entry) { return new Slot(key, lo, hi, entry); }

View File

@@ -0,0 +1,72 @@
package puzzle;
import com.google.gson.Gson;
import lombok.val;
import puzzle.SwedishGenerator.Lemma;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.stream.IntStream;
public class Meta {
static final Gson GSON = new Gson();
private static final int VERSION = 1;
static record ShardLem(long w, int simpel, String[] clues) { }
static final int SHARD_MAGIC = 0x49445831; // "IDX1"
static ShardLem readRecord(Path shardFile, int i) {
try (FileChannel ch = FileChannel.open(shardFile, StandardOpenOption.READ)) {
ByteBuffer hdr = ByteBuffer.allocate(12);
ch.read(hdr);
hdr.flip();
int magic = hdr.getInt();
int ver = hdr.getInt();
int n = hdr.getInt();
if (magic != SHARD_MAGIC || ver != VERSION) throw new IOException("Bad shard");
if (i < 0 || i >= n) throw new IndexOutOfBoundsException();
long tableStart = 12L;
long dataStart = 12L + (long) n * 4L;
int offI = readIntAt(ch, tableStart + (long) i * 4L);
int offIp = (i + 1 < n) ? readIntAt(ch, tableStart + (long) (i + 1) * 4L)
: (int) (ch.size() - dataStart);
int len = offIp - offI;
ByteBuffer buf = ByteBuffer.allocate(len);
ch.position(dataStart + offI);
ch.read(buf);
buf.flip();
var string = StandardCharsets.UTF_8.decode(buf).toString();
val parts = string.split("\t", 3);
return new ShardLem(Lemma.pack(parts[0]), Integer.parseInt(parts[1]), GSON.fromJson(parts[2], String[].class));
} catch (Exception e) {
e.printStackTrace();
return new ShardLem(Lemma.pack("XXX"), -1, new String[0]);
}
}
static final Path[] SHARDS = IntStream.range(0, 10).mapToObj(sId -> Path.of("/home/mike/dev/puzzle-generator/src/main/generated-sources/puzzle").resolve(sId + ".idx")).toArray(
Path[]::new);
static Path shardKey(long word) {
int L = Lemma.length(word);
return SHARDS[L];
}
static String shardKey(String word) {
int L = word.length();
char ch = word.charAt(0);
if (ch < 'A' || ch > 'Z') ch = '_';
///return "" + L + ch; // e.g. "6Z"
return "" + L; // e.g. "6Z"
}
static int readIntAt(FileChannel ch, long pos) throws IOException {
ByteBuffer b = ByteBuffer.allocate(4);
ch.position(pos);
ch.read(b);
b.flip();
return b.getInt();
}
}

View File

@@ -114,8 +114,14 @@ public class SwedishGenerator {
x = y;
return y;
}
public int randint2bit() { return nextU32() & 3; }
public byte randint2bitByte() { return (byte) (nextU32() & 3); }
public int randint2bit() { return nextU32() & 3; }
public byte randint2bitByte() {
var b = (byte) (nextU32() & 3);
/*if (b == 3) {
return 1;
}*/
return b;
}
public int randint(int max) { return (int) (((nextU32() & 0xFFFFFFFFL) % ((long) max - 0L + 1L))); }
public int randint0_SIZE() { return (int) (((nextU32() & 0xFFFFFFFFL) % RANGE_0_SIZE)); }
public int randint0_624() { return (int) (((nextU32() & 0xFFFFFFFFL) % RANGE_0_624)); }
@@ -166,9 +172,10 @@ public class SwedishGenerator {
static class Assign {
long w;
int shardIdx;
}
static record Slotinfo(int key, long lo, long hi, int score, Assign assign, DictEntry entry) {
public static record Slotinfo(int key, long lo, long hi, int score, Assign assign, DictEntry entry) {
public static int wordCount(int k, Slotinfo[] arr) {
for (var n = 1; n < arr.length; n++) if (arr[n].assign.w != X) k++;
@@ -380,15 +387,17 @@ public class SwedishGenerator {
for (var t = 0; t < tries; t++) {
var r = rng.nextFloat();
//int idxInArray = rng.biasedIndexPow3(L - 1);
var w = entry.words[idxs[(int) (r * r * r * (L - 1))]];
var lemIdx = Lemma.unpackIndex(w);
var arrIndex = (int) (r * r * r * (L - 1));
var w = entry.words[idxs[arrIndex]];
var lemIdx = Lemma.unpackIndex(w);
if (Bit1029.get(used, lemIdx)) continue;
low = glo;
top = ghi;
if (!placeWord(k, slo, shi, w)) continue;
Bit1029.set(used, lemIdx);
s.assign.w = w;
s.assign.w = w;
s.assign.shardIdx = arrIndex;
if (backtrack(depth + 1)) return true;
s.assign.w = X;
Bit1029.clear(used, lemIdx);
@@ -403,16 +412,18 @@ public class SwedishGenerator {
var tries = Math.min(MAX_TRIES_PER_SLOT, N);
for (var t = 0; t < tries; t++) {
double r = rng.nextFloat();
var w = entry.words[(int) (r * r * r * (N - 1))];
var lemIdx = Lemma.unpackIndex(w);
double r = rng.nextFloat();
var shardIndx = (int) (r * r * r * (N - 1));
var w = entry.words[shardIndx];
var lemIdx = Lemma.unpackIndex(w);
if (Bit1029.get(used, lemIdx)) continue;
low = glo;
top = ghi;
if (!placeWord(k, slo, shi, w)) continue;
Bit1029.set(used, lemIdx);
s.assign.w = w;
s.assign.w = w;
s.assign.shardIdx = shardIndx;
if (backtrack(depth + 1)) return true;
s.assign.w = X;
Bit1029.clear(used, lemIdx);