tour/puzzle-generator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

introduce bitloops

Browse Source
This commit is contained in:
mike
2026-01-21 06:12:05 +01:00
parent 386777e576
commit ebcbc9b33c
13 changed files with 35 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
src/test/java/puzzle/DictJavaGeneratorMulti.java
View File

@@ -12,62 +12,6 @@ import static java.nio.charset.StandardCharsets.US_ASCII;
public final class DictJavaGeneratorMulti {
// Smaller = more files, but safer for javac/class limits.
private static final int WORDS_CHUNK = 8_192;
private static final int POS_CHUNK = 8_192;
public static void main(String[] args) throws Exception {
var THRESS = 900;
var wordsFile = Path.of(args.length > 0 ? args[0] : "nl_score_hints_v4.csv");
var outDir = Path.of(args.length > 1 ? args[1] : "src/main/generated-sources/puzzle/dict" + THRESS);
var pkg = "puzzle.dict" + THRESS;
var builders = new HashMap<String, ShardBuilder>(16);
var dict = buildDict(wordsFile, builders, THRESS);
Files.createDirectories(outDir);
// Generate L2..L8
for (var L = 2; L <= 8; L++) {
var entry = dict.index()[L];
if (entry == null || entry.words() == null || entry.words().length == 0) {
throw new IllegalStateException("No words for length " + L);
}
writeLengthBundle(outDir, pkg, L, entry);
}
// Aggregator
writeAggregator(outDir, pkg, "DictData", dict.length(), THRESS);
System.out.println("Generated sources into: " + outDir.toAbsolutePath());
}
static String shardKey(long word) {
return "" + Lemma.unpackSize(word) + 1;
}
private static SwedishGenerator.Dict buildDict(Path wordsPath, HashMap<String, ShardBuilder> builders, int thress) throws IOException {
var map = new LongArrayList(100_000);
try (var lines = Files.lines(wordsPath, StandardCharsets.UTF_8)) {
lines.forEach(line -> {
var parts = line.split(",", 4);
var word = parts[0].trim();
var w = SwedishGenerator.Lemma.from(word.getBytes(US_ASCII));
if (!word.equals(SwedishGenerator.Lemma.asWord(w, Export.BYTES.get()))) {
throw new RuntimeException();
}
var score = Integer.parseInt(parts[1].trim());
var simpel = Integer.parseInt(parts[CsvIndexService.SIMPEL_IDX].trim());
if (score < 1 || simpel > thress) {
if (Main.VERBOSE) System.err.println("Word too complex: " + line);
return;
}
var key = shardKey(w);
var sb = builders.computeIfAbsent(key, k -> new ShardBuilder());
map.add(Lemma.pack(w, sb.addRecord()));
});
}
return Dicts.makeDict(map.toArray());
}
interface Dicts {
@@ -107,163 +51,6 @@ public final class DictJavaGeneratorMulti {
}
}
static final class ShardBuilder {
int c;
int addRecord() { return c++; }
}
private static void writeAggregator(Path outDir, String pkg, String cls, int totalLen, int thress) throws IOException {
var out = outDir.resolve(cls + ".java");
try (var w = writer(out)) {
w.write("package " + pkg + ";\n\n");
w.write("public final class " + cls + " {\n");
w.write(" private " + cls + "() {}\n\n");
w.write(" public static final puzzle.SwedishGenerator.Dict DICT" + thress + " = build();\n\n");
w.write(" private static puzzle.SwedishGenerator.Dict build() {\n");
w.write(" puzzle.SwedishGenerator.DictEntry[] idx = new puzzle.SwedishGenerator.DictEntry[puzzle.SwedishGenerator.MAX_WORD_LENGTH_PLUS_ONE];\n");
for (var L = 2; L <= 8; L++) w.write(" idx[" + L + "] = DictDataL" + L + ".entry();\n");
w.write(" return new puzzle.SwedishGenerator.Dict(idx, " + totalLen + ");\n");
w.write(" }\n");
w.write("}\n");
}
}
private static void writeLengthBundle(Path outDir, String pkg, int L, SwedishGenerator.DictEntry e) throws IOException {
var words = e.words();
// flatten posBitsets: [rows][cols] -> flat[]
var bs = e.posBitsets();
var rows = bs.length;
var cols = bs[0].length;
var flat = new long[rows * cols];
var t = 0;
for (var r = 0; r < rows; r++) {
System.arraycopy(bs[r], 0, flat, t, cols);
t += cols;
}
var base = "DictDataL" + L;
// 1) chunk classes
var wChunks = writeChunkClasses(outDir, pkg, base + "W", words, WORDS_CHUNK);
var pChunks = writeChunkClasses(outDir, pkg, base + "P", flat, POS_CHUNK);
// 2) assembler class
writeLengthAssembler(outDir, pkg, base, L, rows, cols, words.length, flat.length, wChunks, pChunks);
}
/** Writes classes like Prefix0..PrefixN each with static long[] DATA. Returns chunk count. */
private static int writeChunkClasses(Path outDir, String pkg, String prefix, long[] data, int chunkSize) throws IOException {
var chunks = (data.length + chunkSize - 1) / chunkSize;
for (var ci = 0; ci < chunks; ci++) {
var from = ci * chunkSize;
var to = Math.min(data.length, from + chunkSize);
var out = outDir.resolve(prefix + ci + ".java");
try (var w = writer(out)) {
w.write("package " + pkg + ";\n\n");
w.write("public final class " + prefix + ci + " {\n");
w.write(" private " + prefix + ci + "() {}\n");
w.write(" public static long[] get() {\n");
w.write(" return new long[] { \n");
for (var i = from; i < to; i++) {
w.write(" " + toLongLiteral(data[i]) + (i + 1 < to ? "," : "") + "\n");
}
w.write(" };\n");
w.write(" }\n");
w.write("}\n");
}
}
return chunks;
}
private static void writeLengthAssembler(Path outDir, String pkg, String cls, int L,
int rows, int cols,
int wordsLen, int posLen,
int wChunks, int pChunks) throws IOException {
var out = outDir.resolve(cls + ".java");
try (var w = writer(out)) {
w.write("package " + pkg + ";\n\n");
w.write("public final class " + cls + " {\n");
w.write(" private " + cls + "() {}\n\n");
w.write(" static final int LEN = " + L + ";\n");
w.write(" static final int ROWS = " + rows + ";\n");
w.write(" static final int COLS = " + cols + ";\n");
w.write(" static final int WORDS_LEN = " + wordsLen + ";\n");
w.write(" static final int POS_LEN = " + posLen + ";\n\n");
// assemble words
w.write(" private static long[] words() {\n");
var wPrefix = "DictDataL" + L + "W";
if (wChunks == 1) {
w.write(" return " + wPrefix + "0.get();\n");
} else {
w.write(" long[] out = new long[WORDS_LEN];\n");
w.write(" int k = 0;\n");
for (var ci = 0; ci < wChunks; ci++) {
w.write(" k = copy(out, k, " + wPrefix + ci + ".get());\n");
}
w.write(" return out;\n");
}
w.write(" }\n\n");
// assemble pos
w.write(" private static long[] posFlat() {\n");
var pPrefix = "DictDataL" + L + "P";
if (pChunks == 1) {
w.write(" return " + pPrefix + "0.get();\n");
} else {
w.write(" long[] out = new long[POS_LEN];\n");
w.write(" int k = 0;\n");
for (var ci = 0; ci < pChunks; ci++) {
w.write(" k = copy(out, k, " + pPrefix + ci + ".get());\n");
}
w.write(" return out;\n");
}
w.write(" }\n\n");
// entry
w.write(" public static puzzle.SwedishGenerator.DictEntry entry() {\n");
w.write(" long[] wds = words();\n");
w.write(" long[] flat = posFlat();\n");
w.write(" long[][] pos = reshape(flat, ROWS, COLS);\n");
w.write(" return new puzzle.SwedishGenerator.DictEntry(wds, pos, wds.length, (wds.length + 63) >>> 6);\n");
w.write(" }\n\n");
// helpers
w.write(" private static int copy(long[] dst, int at, long[] src) {\n");
w.write(" System.arraycopy(src, 0, dst, at, src.length);\n");
w.write(" return at + src.length;\n");
w.write(" }\n\n");
w.write(" private static long[][] reshape(long[] flat, int rows, int cols) {\n");
w.write(" long[][] out = new long[rows][cols];\n");
w.write(" int k = 0;\n");
w.write(" for (int r = 0; r < rows; r++) {\n");
w.write(" System.arraycopy(flat, k, out[r], 0, cols);\n");
w.write(" k += cols;\n");
w.write(" }\n");
w.write(" return out;\n");
w.write(" }\n");
w.write("}\n");
}
}
private static BufferedWriter writer(Path out) throws IOException {
return Files.newBufferedWriter(out, StandardCharsets.UTF_8,
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
}
private static String toLongLiteral(long v) { return "0x" + Long.toUnsignedString(v, 16) + "L"; }
public static final class CsvIndexService {
static int SIMPEL_IDX = 2;
}
record DictEntryDTO(LongArrayList words, IntListDTO[][] pos) {