introduce bitloops
This commit is contained in:
mike
254
src/test/java/puzzle/CsvIndexService.java
Normal file
254
src/test/java/puzzle/CsvIndexService.java
Normal file
@@ -0,0 +1,254 @@
|
||||
package puzzle;
|
||||
|
||||
import puzzle.SwedishGenerator.Lemma;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.Closeable;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Arrays;
|
||||
import java.util.function.LongConsumer;
|
||||
import static java.nio.charset.StandardCharsets.US_ASCII;
|
||||
|
||||
public final class CsvIndexService
|
||||
implements Closeable {
|
||||
|
||||
static final ScopedValue<CsvIndexService> SC = ScopedValue.newInstance();
|
||||
private static final int MAGIC = 0x4C494458; // "LIDX"
|
||||
private static final int VERSION = 1;
|
||||
static int SIMPEL_IDX = 3;
|
||||
private final Path csvPath;
|
||||
private final Path idxPath;
|
||||
|
||||
private volatile long[] offsets; // lazy
|
||||
private volatile FileChannel csvChannel; // open once
|
||||
private final Object lock = new Object();
|
||||
|
||||
public CsvIndexService(Path csvPath, Path idxPath) {
|
||||
this.csvPath = csvPath;
|
||||
this.idxPath = idxPath;
|
||||
}
|
||||
public static int lineToSimpel(String line) {
|
||||
var parts = line.split(",", 5);
|
||||
return Integer.parseInt(parts[SIMPEL_IDX].trim());
|
||||
}
|
||||
public static String[] lineToClue(String line) {
|
||||
if (line.isBlank()) throw new RuntimeException("Empty line");
|
||||
var parts = line.split(",", 5);
|
||||
var rawClue = parts[4].trim();
|
||||
if (rawClue.startsWith("\"") && rawClue.endsWith("\"")) {
|
||||
rawClue = rawClue.substring(1, rawClue.length() - 1).replace("\"\"", "\"");
|
||||
}
|
||||
return Meta.GSON.fromJson(rawClue, String[].class);
|
||||
}
|
||||
public static void lineToLemma(String line, LongConsumer ok) {
|
||||
if (line.isBlank()) {
|
||||
throw new RuntimeException("Empty line");
|
||||
}
|
||||
var parts = line.split(",", 5);
|
||||
var id = Integer.parseInt(parts[0].trim());
|
||||
var word = parts[1].trim();
|
||||
int score = Integer.parseInt(parts[2].trim());
|
||||
if (score < 1) {
|
||||
if (Main.VERBOSE) System.err.println("Word too complex: " + line);
|
||||
return;
|
||||
}
|
||||
ok.accept(Lemma.pack(id, word.getBytes(US_ASCII)));
|
||||
}
|
||||
|
||||
public static int simpel(int index) {
|
||||
try {
|
||||
if (SC.isBound())
|
||||
return lineToSimpel(SC.get().getLine(index));
|
||||
return -1;
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Failed to get clues for index " + index, e);
|
||||
}
|
||||
}
|
||||
public static String[] clues(int index) {
|
||||
try {
|
||||
if (SC.isBound())
|
||||
return lineToClue(SC.get().getLine(index));
|
||||
return new String[0];
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Failed to get clues for index " + index, e);
|
||||
}
|
||||
}
|
||||
/** Haal één regel op (0-based line index), met self-healing index (1x rebuild). */
|
||||
public String getLine(int lineIndex) throws IOException {
|
||||
ensureLoaded();
|
||||
|
||||
var line = readLineAt(lineIndex);
|
||||
|
||||
if (startsWithIndex(line, lineIndex)) return line;
|
||||
|
||||
// mismatch => rebuild index en nog 1x proberen
|
||||
synchronized (lock) {
|
||||
rebuildIndexLocked();
|
||||
line = readLineAt(lineIndex);
|
||||
if (startsWithIndex(line, lineIndex)) return line;
|
||||
}
|
||||
|
||||
throw new RuntimeException("Index mismatch after rebuild. Requested=" + lineIndex + ", got line=" + preview(line));
|
||||
}
|
||||
|
||||
public void ensureLoaded() throws IOException {
|
||||
if (offsets != null && csvChannel != null && csvChannel.isOpen()) return;
|
||||
|
||||
synchronized (lock) {
|
||||
if (offsets != null && csvChannel != null && csvChannel.isOpen()) return;
|
||||
|
||||
csvChannel = FileChannel.open(csvPath, StandardOpenOption.READ);
|
||||
|
||||
if (Files.exists(idxPath)) {
|
||||
try {
|
||||
offsets = readIndex(idxPath);
|
||||
return;
|
||||
} catch (IOException badIndex) {
|
||||
// fall-through -> rebuild
|
||||
}
|
||||
}
|
||||
|
||||
rebuildIndexLocked();
|
||||
}
|
||||
}
|
||||
|
||||
private void rebuildIndexLocked() throws IOException {
|
||||
var built = buildOffsets(csvPath);
|
||||
writeIndex(idxPath, built);
|
||||
offsets = built;
|
||||
}
|
||||
|
||||
private String readLineAt(int lineIndex) throws IOException {
|
||||
var local = offsets;
|
||||
if (lineIndex < 0 || lineIndex >= local.length) {
|
||||
throw new IndexOutOfBoundsException("lineIndex=" + lineIndex + ", max=" + (local.length - 1));
|
||||
}
|
||||
|
||||
long currentPos = local[lineIndex];
|
||||
|
||||
// lees in blokjes (sneller dan 1 byte) tot newline
|
||||
var buf = new byte[8192];
|
||||
var total = 0;
|
||||
var out = new byte[256];
|
||||
|
||||
while (true) {
|
||||
var bb = ByteBuffer.wrap(buf);
|
||||
var n = csvChannel.read(bb, currentPos);
|
||||
if (n < 0) break; // EOF
|
||||
currentPos += n;
|
||||
var end = n;
|
||||
|
||||
for (var i = 0; i < end; i++) {
|
||||
var b = buf[i];
|
||||
|
||||
if (b == (byte) '\n') {
|
||||
return new String(out, 0, total, StandardCharsets.UTF_8);
|
||||
}
|
||||
if (b == (byte) '\r') continue;
|
||||
|
||||
if (total == out.length) out = Arrays.copyOf(out, out.length * 2);
|
||||
out[total++] = b;
|
||||
}
|
||||
}
|
||||
|
||||
return new String(out, 0, total, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/** Check: begint de regel met "<lineIndex>," */
|
||||
private static boolean startsWithIndex(String line, int lineIndex) {
|
||||
if (line == null || line.isEmpty()) return false;
|
||||
|
||||
var comma = line.indexOf(',');
|
||||
if (comma <= 0) return false;
|
||||
|
||||
// snelle parse zonder split
|
||||
long v = 0;
|
||||
for (var i = 0; i < comma; i++) {
|
||||
var c = line.charAt(i);
|
||||
if (c < '0' || c > '9') return false;
|
||||
v = (v * 10) + (c - '0');
|
||||
if (v > Integer.MAX_VALUE) return false;
|
||||
}
|
||||
return v == lineIndex;
|
||||
}
|
||||
|
||||
private static String preview(String s) {
|
||||
if (s == null) return "null";
|
||||
return s.length() <= 120 ? s : s.substring(0, 120) + "...";
|
||||
}
|
||||
|
||||
/** Bouw offsets door newlines te scannen. Resultaat is exact getrimd. */
|
||||
public static long[] buildOffsets(Path path) throws IOException {
|
||||
try (var ch = FileChannel.open(path, StandardOpenOption.READ)) {
|
||||
var offs = new long[131072]; // start-capacity, groeit indien nodig
|
||||
var c = 0;
|
||||
offs[c++] = 0;
|
||||
|
||||
var buf = ByteBuffer.allocateDirect(1 << 20);
|
||||
int pos = 0;
|
||||
|
||||
while (true) {
|
||||
buf.clear();
|
||||
var n = ch.read(buf);
|
||||
if (n < 0) break;
|
||||
buf.flip();
|
||||
|
||||
for (var i = 0; i < n; i++) {
|
||||
if (buf.get(i) == (byte) '\n') {
|
||||
if (c == offs.length) offs = Arrays.copyOf(offs, offs.length * 2);
|
||||
offs[c++] = pos + i + 1;
|
||||
}
|
||||
}
|
||||
pos += n;
|
||||
}
|
||||
|
||||
return Arrays.copyOf(offs, c);
|
||||
}
|
||||
}
|
||||
|
||||
public static void writeIndex(Path out, long[] offsets) throws IOException {
|
||||
try (var dos = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(
|
||||
out, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)))) {
|
||||
dos.writeInt(MAGIC);
|
||||
dos.writeInt(VERSION);
|
||||
dos.writeInt(offsets.length);
|
||||
for (var v : offsets) dos.writeLong(v);
|
||||
}
|
||||
}
|
||||
|
||||
public static long[] readIndex(Path in) throws IOException {
|
||||
try (var dis = new DataInputStream(new BufferedInputStream(Files.newInputStream(in)))) {
|
||||
if (dis.readInt() != MAGIC) throw new IOException("Not a LIDX file");
|
||||
|
||||
var version = dis.readInt();
|
||||
if (version != VERSION) throw new IOException("Unsupported version: " + version);
|
||||
|
||||
var n = dis.readInt();
|
||||
if (n < 0) throw new IOException("Corrupt length: " + n);
|
||||
|
||||
var offsets = new long[n];
|
||||
for (var i = 0; i < n; i++) offsets[i] = dis.readLong();
|
||||
return offsets;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
synchronized (lock) {
|
||||
if (csvChannel != null) csvChannel.close();
|
||||
csvChannel = null;
|
||||
offsets = null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
193
src/test/java/puzzle/DictCodeGen.java
Normal file
193
src/test/java/puzzle/DictCodeGen.java
Normal file
@@ -0,0 +1,193 @@
|
||||
package puzzle;
|
||||
|
||||
import lombok.val;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import puzzle.Export.Dicts;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
|
||||
public final class DictCodeGen {
|
||||
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
DictJavaGenerator.main(args); // gebruikt jouw makeDict logic
|
||||
}
|
||||
/**
|
||||
* Generates Java source files for dictionary data, split by word length (2..8),
|
||||
* and further chunked to avoid "code too large" / constant pool issues.
|
||||
*
|
||||
* Output:
|
||||
* - DictDataL2.java .. DictDataL8.java (arrays chunked)
|
||||
* - DictData.java (aggregator that builds Dict)
|
||||
*
|
||||
* Usage:
|
||||
* java puzzle.codegen.DictJavaGenerator <wordsFile> <outDir> <packageName>
|
||||
*
|
||||
* Example:
|
||||
* java puzzle.codegen.DictJavaGenerator nl_score_hints_v3.csv src/main/java puzzle
|
||||
*/
|
||||
public final class DictJavaGenerator {
|
||||
|
||||
// tune if needed
|
||||
private static final int WORDS_CHUNK = 8_192>>>5; // longs per chunk
|
||||
private static final int POS_CHUNK = 8_192>>>5; // longs per chunk
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Path wordsFile = Path.of(args.length > 0 ? args[0] : "nl_score_hints_v3.csv");
|
||||
Path outDir = Path.of(args.length > 1 ? args[1] : "/home/mike/dev/puzzle-generator/src/main/generated-sources/puzzle");
|
||||
String pkg = "puzzle";
|
||||
|
||||
SwedishGenerator.Dict dict = buildDict(wordsFile);
|
||||
|
||||
Files.createDirectories(outDir);
|
||||
|
||||
// emit L2..L8
|
||||
for (int L = 2; L <= 8; L++) {
|
||||
var entry = dict.index()[L];
|
||||
if (entry == null || entry.words() == null || entry.words().length == 0) {
|
||||
throw new IllegalStateException("No words for length " + L);
|
||||
}
|
||||
writeLengthClass(outDir, pkg, "DictDataL" + L, L, entry);
|
||||
}
|
||||
|
||||
// emit aggregator
|
||||
writeAggregator(outDir, pkg, "DictData", dict.length());
|
||||
|
||||
System.out.println("Generated dictionary sources into: " + outDir.toAbsolutePath());
|
||||
}
|
||||
|
||||
private static SwedishGenerator.Dict buildDict(Path wordsPath) throws IOException {
|
||||
var map = new LongArrayList(100_000);
|
||||
try (var lines = Files.lines(wordsPath, StandardCharsets.UTF_8)) {
|
||||
lines.forEach(line -> CsvIndexService.lineToLemma(line, map::add));
|
||||
}
|
||||
return Dicts.makeDict(map.toArray());
|
||||
}
|
||||
|
||||
private static void writeAggregator(Path outDir, String pkg, String cls, int totalLen) throws IOException {
|
||||
Path out = outDir.resolve(cls + ".java");
|
||||
try (BufferedWriter w = Files.newBufferedWriter(out, StandardCharsets.UTF_8,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
|
||||
|
||||
w.write("package " + pkg + ";\n\n");
|
||||
w.write("public final class " + cls + " {\n");
|
||||
w.write(" private " + cls + "() {}\n\n");
|
||||
w.write(" public static final SwedishGenerator.Dict DICT = build();\n\n");
|
||||
w.write(" private static SwedishGenerator.Dict build() {\n");
|
||||
w.write(" SwedishGenerator.DictEntry[] idx = new SwedishGenerator.DictEntry[SwedishGenerator.MAX_WORD_LENGTH_PLUS_ONE];\n");
|
||||
w.write(" idx[2] = DictDataL2.entry();\n");
|
||||
w.write(" idx[3] = DictDataL3.entry();\n");
|
||||
w.write(" idx[4] = DictDataL4.entry();\n");
|
||||
w.write(" idx[5] = DictDataL5.entry();\n");
|
||||
w.write(" idx[6] = DictDataL6.entry();\n");
|
||||
w.write(" idx[7] = DictDataL7.entry();\n");
|
||||
w.write(" idx[8] = DictDataL8.entry();\n");
|
||||
w.write(" return new SwedishGenerator.Dict(idx, " + totalLen + ");\n");
|
||||
w.write(" }\n");
|
||||
w.write("}\n");
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeLengthClass(Path outDir, String pkg, String cls, int L, SwedishGenerator.DictEntry e) throws IOException {
|
||||
Path out = outDir.resolve(cls + ".java");
|
||||
try (BufferedWriter w = Files.newBufferedWriter(out, StandardCharsets.UTF_8,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) {
|
||||
|
||||
w.write("package " + pkg + ";\n\n");
|
||||
w.write("public final class " + cls + " {\n");
|
||||
w.write(" private " + cls + "() {}\n\n");
|
||||
|
||||
long[] words = e.words();
|
||||
|
||||
// flatten posBitsets: [rows][cols] -> flat[]
|
||||
long[][] bs = e.posBitsets();
|
||||
int rows = bs.length;
|
||||
int cols = bs[0].length;
|
||||
long[] flat = new long[rows * cols];
|
||||
int t = 0;
|
||||
for (int r = 0; r < rows; r++) {
|
||||
System.arraycopy(bs[r], 0, flat, t, cols);
|
||||
t += cols;
|
||||
}
|
||||
|
||||
w.write(" static final int LEN = " + L + ";\n");
|
||||
w.write(" static final int ROWS = " + rows + ";\n");
|
||||
w.write(" static final int COLS = " + cols + ";\n");
|
||||
w.write(" static final int WORDS_LEN = " + words.length + ";\n");
|
||||
w.write(" static final int POS_LEN = " + flat.length + ";\n\n");
|
||||
|
||||
// chunked arrays
|
||||
int wordChunks = emitLongArrayChunked(w, "WORDS", words, WORDS_CHUNK);
|
||||
int posChunks = emitLongArrayChunked(w, "POS", flat, POS_CHUNK);
|
||||
|
||||
// joiners
|
||||
emitJoiner(w, "WORDS", "WORDS", words.length, wordChunks);
|
||||
emitJoiner(w, "POS", "POS", flat.length, posChunks);
|
||||
|
||||
// entry builder
|
||||
w.write(" public static SwedishGenerator.DictEntry entry() {\n");
|
||||
w.write(" long[] words = WORDS();\n");
|
||||
w.write(" long[] flat = POS();\n");
|
||||
w.write(" long[][] pos = reshape(flat, ROWS, COLS);\n");
|
||||
w.write(" return new SwedishGenerator.DictEntry(words, pos, words.length, (words.length + 63) >>> 6);\n");
|
||||
w.write(" }\n\n");
|
||||
|
||||
// helpers
|
||||
w.write(" private static int copy(long[] dst, int at, long[] src) {\n");
|
||||
w.write(" System.arraycopy(src, 0, dst, at, src.length);\n");
|
||||
w.write(" return at + src.length;\n");
|
||||
w.write(" }\n\n");
|
||||
|
||||
w.write(" private static long[][] reshape(long[] flat, int rows, int cols) {\n");
|
||||
w.write(" long[][] out = new long[rows][cols];\n");
|
||||
w.write(" int k = 0;\n");
|
||||
w.write(" for (int r = 0; r < rows; r++) {\n");
|
||||
w.write(" System.arraycopy(flat, k, out[r], 0, cols);\n");
|
||||
w.write(" k += cols;\n");
|
||||
w.write(" }\n");
|
||||
w.write(" return out;\n");
|
||||
w.write(" }\n");
|
||||
|
||||
w.write("}\n");
|
||||
}
|
||||
}
|
||||
|
||||
/** Emits baseName_0..k arrays and returns chunkCount. */
|
||||
private static int emitLongArrayChunked(BufferedWriter w, String baseName, long[] data, int chunkSize) throws IOException {
|
||||
int chunks = (data.length + chunkSize - 1) / chunkSize;
|
||||
for (int ci = 0; ci < chunks; ci++) {
|
||||
int from = ci * chunkSize;
|
||||
int to = Math.min(data.length, from + chunkSize);
|
||||
|
||||
w.write(" static final long[] " + baseName + "_" + ci + " = new long[] {\n");
|
||||
for (int i = from; i < to; i++) {
|
||||
w.write(" " + toLongLiteral(data[i]) + (i + 1 < to ? "," : "") + "\n");
|
||||
}
|
||||
w.write(" };\n\n");
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
private static void emitJoiner(BufferedWriter w, String funcName, String baseName, int totalLen, int chunks) throws IOException {
|
||||
w.write(" static long[] " + funcName + "() {\n");
|
||||
w.write(" long[] out = new long[" + totalLen + "];\n");
|
||||
w.write(" int k = 0;\n");
|
||||
for (int ci = 0; ci < chunks; ci++) {
|
||||
w.write(" k = copy(out, k, " + baseName + "_" + ci + ");\n");
|
||||
}
|
||||
w.write(" return out;\n");
|
||||
w.write(" }\n\n");
|
||||
}
|
||||
|
||||
private static String toLongLiteral(long v) {
|
||||
// compact unsigned hex literal
|
||||
return "0x" + Long.toUnsignedString(v, 16) + "L";
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
279
src/test/java/puzzle/DictJavaGeneratorMulti.java
Normal file
279
src/test/java/puzzle/DictJavaGeneratorMulti.java
Normal file
@@ -0,0 +1,279 @@
|
||||
package puzzle;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import puzzle.Export.Dicts;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.*;
|
||||
import java.util.Arrays;
|
||||
|
||||
public final class DictJavaGeneratorMulti {
|
||||
|
||||
// Smaller = more files, but safer for javac/class limits.
|
||||
private static final int WORDS_CHUNK = 8_192;
|
||||
private static final int POS_CHUNK = 8_192;
|
||||
@Test
|
||||
public void dictCodeGen15() {
|
||||
System.out.println(DictData.DICT);
|
||||
}
|
||||
public static void main(String[] args) throws Exception {
|
||||
Path wordsFile = Path.of(args.length > 0 ? args[0] : "nl_score_hints_v3.csv");
|
||||
Path outDir = Path.of(args.length > 1 ? args[1] : "/home/mike/dev/puzzle-generator/src/main/generated-sources/puzzle");
|
||||
String pkg = "puzzle";
|
||||
|
||||
SwedishGenerator.Dict dict = buildDict(wordsFile);
|
||||
|
||||
Files.createDirectories(outDir);
|
||||
|
||||
// Generate L2..L8
|
||||
for (int L = 2; L <= 8; L++) {
|
||||
var entry = dict.index()[L];
|
||||
if (entry == null || entry.words() == null || entry.words().length == 0) {
|
||||
throw new IllegalStateException("No words for length " + L);
|
||||
}
|
||||
writeLengthBundle(outDir, pkg, L, entry);
|
||||
}
|
||||
|
||||
// Aggregator
|
||||
writeAggregator(outDir, pkg, "DictData", dict.length());
|
||||
generateHintShards(wordsFile, outDir);
|
||||
System.out.println("Generated sources into: " + outDir.toAbsolutePath());
|
||||
}
|
||||
|
||||
private static SwedishGenerator.Dict buildDict(Path wordsPath) throws IOException {
|
||||
var map = new LongArrayList(100_000);
|
||||
try (var lines = Files.lines(wordsPath, StandardCharsets.UTF_8)) {
|
||||
lines.forEach(line -> CsvIndexService.lineToLemma(line, map::add));
|
||||
}
|
||||
return Dicts.makeDict(map.toArray());
|
||||
}
|
||||
|
||||
static final int VERSION = 1;
|
||||
static String wordFromLine(String line) {
|
||||
// ID,WORD,*,*,"JSON"
|
||||
var parts = line.split(",", 5);
|
||||
return parts[1].trim();
|
||||
}
|
||||
static final class IntArrayList {
|
||||
|
||||
int[] a;
|
||||
int size;
|
||||
IntArrayList(int cap) { a = new int[cap]; }
|
||||
void add(int v) {
|
||||
if (size == a.length) a = Arrays.copyOf(a, a.length * 2);
|
||||
a[size++] = v;
|
||||
}
|
||||
int size() { return size; }
|
||||
int get(int i) { return a[i]; }
|
||||
int[] toArray() { return Arrays.copyOf(a, size); }
|
||||
}
|
||||
|
||||
static final class ShardBuilder {
|
||||
|
||||
final IntArrayList offsets = new IntArrayList(4096);
|
||||
final ByteArrayOutputStream data = new ByteArrayOutputStream(1 << 20); // grows
|
||||
void addRecord(byte[] rec) throws IOException {
|
||||
offsets.add(data.size());
|
||||
data.write(rec);
|
||||
}
|
||||
}
|
||||
static void generateHintShards(Path csv, Path outDir) throws IOException {
|
||||
Files.createDirectories(outDir);
|
||||
|
||||
var builders = new java.util.HashMap<String, ShardBuilder>(256);
|
||||
|
||||
try (var lines = Files.lines(csv, StandardCharsets.UTF_8)) {
|
||||
lines.forEach(line -> {
|
||||
if (line == null || line.isBlank()) return;
|
||||
|
||||
String word = wordFromLine(line);
|
||||
String[] clues = CsvIndexService.lineToClue(line);
|
||||
int simpel = CsvIndexService.lineToSimpel(line);
|
||||
|
||||
// serialize to: WORD \t JSON \n
|
||||
// (als je al JSON string wilt bewaren: gebruik Gson.toJson(clues))
|
||||
String json = Meta.GSON.toJson(clues);
|
||||
String recStr = word + "\t" + simpel + "\t" + json + "\n";
|
||||
byte[] rec = recStr.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
String key = Meta.shardKey(word);
|
||||
ShardBuilder sb = builders.computeIfAbsent(key, k -> new ShardBuilder());
|
||||
try {
|
||||
sb.addRecord(rec);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
});
|
||||
} catch (UncheckedIOException uioe) {
|
||||
throw uioe.getCause();
|
||||
}
|
||||
|
||||
// flush all shards to disk as <key>.idx (e.g. 6Z.idx)
|
||||
for (var e : builders.entrySet()) {
|
||||
writeIndexedShard(outDir.resolve(e.getKey() + ".idx"), e.getValue());
|
||||
}
|
||||
}
|
||||
static void writeIndexedShard(Path out, ShardBuilder sb) throws IOException {
|
||||
int n = sb.offsets.size();
|
||||
int[] offs = sb.offsets.toArray();
|
||||
byte[] data = sb.data.toByteArray();
|
||||
|
||||
try (FileChannel ch = FileChannel.open(out,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING,
|
||||
StandardOpenOption.WRITE)) {
|
||||
|
||||
// header
|
||||
ByteBuffer hdr = ByteBuffer.allocate(12);
|
||||
hdr.putInt(Meta.SHARD_MAGIC).putInt(VERSION).putInt(n).flip();
|
||||
ch.write(hdr);
|
||||
|
||||
// offsets table (int per record)
|
||||
ByteBuffer tbl = ByteBuffer.allocate(n * 4);
|
||||
for (int i = 0; i < n; i++) tbl.putInt(offs[i]);
|
||||
tbl.flip();
|
||||
ch.write(tbl);
|
||||
|
||||
// data
|
||||
ch.write(ByteBuffer.wrap(data));
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeAggregator(Path outDir, String pkg, String cls, int totalLen) throws IOException {
|
||||
Path out = outDir.resolve(cls + ".java");
|
||||
try (BufferedWriter w = writer(out)) {
|
||||
w.write("package " + pkg + ";\n\n");
|
||||
w.write("public final class " + cls + " {\n");
|
||||
w.write(" private " + cls + "() {}\n\n");
|
||||
w.write(" public static final SwedishGenerator.Dict DICT = build();\n\n");
|
||||
w.write(" private static SwedishGenerator.Dict build() {\n");
|
||||
w.write(" SwedishGenerator.DictEntry[] idx = new SwedishGenerator.DictEntry[SwedishGenerator.MAX_WORD_LENGTH_PLUS_ONE];\n");
|
||||
for (int L = 2; L <= 8; L++) w.write(" idx[" + L + "] = DictDataL" + L + ".entry();\n");
|
||||
w.write(" return new SwedishGenerator.Dict(idx, " + totalLen + ");\n");
|
||||
w.write(" }\n");
|
||||
w.write("}\n");
|
||||
}
|
||||
}
|
||||
|
||||
private static void writeLengthBundle(Path outDir, String pkg, int L, SwedishGenerator.DictEntry e) throws IOException {
|
||||
long[] words = e.words();
|
||||
|
||||
// flatten posBitsets: [rows][cols] -> flat[]
|
||||
long[][] bs = e.posBitsets();
|
||||
int rows = bs.length;
|
||||
int cols = bs[0].length;
|
||||
long[] flat = new long[rows * cols];
|
||||
int t = 0;
|
||||
for (int r = 0; r < rows; r++) {
|
||||
System.arraycopy(bs[r], 0, flat, t, cols);
|
||||
t += cols;
|
||||
}
|
||||
|
||||
String base = "DictDataL" + L;
|
||||
|
||||
// 1) chunk classes
|
||||
int wChunks = writeChunkClasses(outDir, pkg, base + "W", words, WORDS_CHUNK);
|
||||
int pChunks = writeChunkClasses(outDir, pkg, base + "P", flat, POS_CHUNK);
|
||||
|
||||
// 2) assembler class
|
||||
writeLengthAssembler(outDir, pkg, base, L, rows, cols, words.length, flat.length, wChunks, pChunks);
|
||||
}
|
||||
|
||||
/** Writes classes like Prefix0..PrefixN each with static final long[] DATA. Returns chunk count. */
|
||||
private static int writeChunkClasses(Path outDir, String pkg, String prefix, long[] data, int chunkSize) throws IOException {
|
||||
int chunks = (data.length + chunkSize - 1) / chunkSize;
|
||||
for (int ci = 0; ci < chunks; ci++) {
|
||||
int from = ci * chunkSize;
|
||||
int to = Math.min(data.length, from + chunkSize);
|
||||
|
||||
Path out = outDir.resolve(prefix + ci + ".java");
|
||||
try (BufferedWriter w = writer(out)) {
|
||||
w.write("package " + pkg + ";\n\n");
|
||||
w.write("public final class " + prefix + ci + " {\n");
|
||||
w.write(" private " + prefix + ci + "() {}\n");
|
||||
w.write(" public static final long[] DATA = new long[] {\n");
|
||||
for (int i = from; i < to; i++) {
|
||||
w.write(" " + toLongLiteral(data[i]) + (i + 1 < to ? "," : "") + "\n");
|
||||
}
|
||||
w.write(" };\n");
|
||||
w.write("}\n");
|
||||
}
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
private static void writeLengthAssembler(Path outDir, String pkg, String cls, int L,
|
||||
int rows, int cols,
|
||||
int wordsLen, int posLen,
|
||||
int wChunks, int pChunks) throws IOException {
|
||||
Path out = outDir.resolve(cls + ".java");
|
||||
try (BufferedWriter w = writer(out)) {
|
||||
w.write("package " + pkg + ";\n\n");
|
||||
w.write("public final class " + cls + " {\n");
|
||||
w.write(" private " + cls + "() {}\n\n");
|
||||
|
||||
w.write(" static final int LEN = " + L + ";\n");
|
||||
w.write(" static final int ROWS = " + rows + ";\n");
|
||||
w.write(" static final int COLS = " + cols + ";\n");
|
||||
w.write(" static final int WORDS_LEN = " + wordsLen + ";\n");
|
||||
w.write(" static final int POS_LEN = " + posLen + ";\n\n");
|
||||
|
||||
// assemble words
|
||||
w.write(" private static long[] words() {\n");
|
||||
w.write(" long[] out = new long[WORDS_LEN];\n");
|
||||
w.write(" int k = 0;\n");
|
||||
for (int ci = 0; ci < wChunks; ci++) {
|
||||
w.write(" k = copy(out, k, DictDataL" + L + "W" + ci + ".DATA);\n");
|
||||
}
|
||||
w.write(" return out;\n");
|
||||
w.write(" }\n\n");
|
||||
|
||||
// assemble pos
|
||||
w.write(" private static long[] posFlat() {\n");
|
||||
w.write(" long[] out = new long[POS_LEN];\n");
|
||||
w.write(" int k = 0;\n");
|
||||
for (int ci = 0; ci < pChunks; ci++) {
|
||||
w.write(" k = copy(out, k, DictDataL" + L + "P" + ci + ".DATA);\n");
|
||||
}
|
||||
w.write(" return out;\n");
|
||||
w.write(" }\n\n");
|
||||
|
||||
// entry
|
||||
w.write(" public static SwedishGenerator.DictEntry entry() {\n");
|
||||
w.write(" long[] wds = words();\n");
|
||||
w.write(" long[] flat = posFlat();\n");
|
||||
w.write(" long[][] pos = reshape(flat, ROWS, COLS);\n");
|
||||
w.write(" return new SwedishGenerator.DictEntry(wds, pos, wds.length, (wds.length + 63) >>> 6);\n");
|
||||
w.write(" }\n\n");
|
||||
|
||||
// helpers
|
||||
w.write(" private static int copy(long[] dst, int at, long[] src) {\n");
|
||||
w.write(" System.arraycopy(src, 0, dst, at, src.length);\n");
|
||||
w.write(" return at + src.length;\n");
|
||||
w.write(" }\n\n");
|
||||
|
||||
w.write(" private static long[][] reshape(long[] flat, int rows, int cols) {\n");
|
||||
w.write(" long[][] out = new long[rows][cols];\n");
|
||||
w.write(" int k = 0;\n");
|
||||
w.write(" for (int r = 0; r < rows; r++) {\n");
|
||||
w.write(" System.arraycopy(flat, k, out[r], 0, cols);\n");
|
||||
w.write(" k += cols;\n");
|
||||
w.write(" }\n");
|
||||
w.write(" return out;\n");
|
||||
w.write(" }\n");
|
||||
|
||||
w.write("}\n");
|
||||
}
|
||||
}
|
||||
|
||||
private static BufferedWriter writer(Path out) throws IOException {
|
||||
return Files.newBufferedWriter(out, StandardCharsets.UTF_8,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
|
||||
}
|
||||
|
||||
private static String toLongLiteral(long v) {
|
||||
return "0x" + Long.toUnsignedString(v, 16) + "L";
|
||||
}
|
||||
}
|
||||
@@ -10,9 +10,11 @@ import puzzle.Export.PuzzleResult;
|
||||
import puzzle.Export.Rewards;
|
||||
import puzzle.SwedishGenerator.Assign;
|
||||
import puzzle.SwedishGenerator.FillResult;
|
||||
import puzzle.SwedishGenerator.Lemma;
|
||||
import puzzle.SwedishGenerator.Slotinfo;
|
||||
import puzzle.SwedishGeneratorTest.Idx;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
@@ -69,7 +71,7 @@ public class ExportFormatTest {
|
||||
|
||||
var fillResult = new FillResult(true, 0, 0, 0, 0, new FillStats());
|
||||
var puzzleResult = new PuzzleResult(new Clued(clues), grid, new Slotinfo[]{
|
||||
new Slotinfo(key, lo, 0L, 0, new Assign(TEST), null)
|
||||
new Slotinfo(key, lo, 0L, 0, new Assign(TEST, 0), null)
|
||||
}, fillResult);
|
||||
|
||||
var rewards = new Rewards(10, 5, 1);
|
||||
@@ -134,5 +136,15 @@ public class ExportFormatTest {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testShardToClue() {
|
||||
val index = 1;
|
||||
val word = DictData.DICT.index()[3].words()[index];
|
||||
val assigned = new Assign(word, index);
|
||||
val lemma = Lemma.unpackIndex(word);
|
||||
var word1 = Lemma.asWord(word);
|
||||
val shard = Meta.shardKey(assigned.w);
|
||||
val clue = Meta.readRecord(shard, index);
|
||||
assertNotNull(clue);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,10 @@ import puzzle.Export.Rewards;
|
||||
import puzzle.Main.Opts;
|
||||
import puzzle.Masker.Clues;
|
||||
import puzzle.SwedishGenerator.Rng;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -43,7 +47,16 @@ public class MainTest {
|
||||
this.tries = 1;
|
||||
this.verbose = false;
|
||||
}};
|
||||
static final Dict dict = Dicts.loadDict(opts.wordsPath);
|
||||
static final Dict dict = loadDict(opts.wordsPath);
|
||||
public static Dict loadDict(String wordsPath) {
|
||||
var map = new LongArrayList(100_000);
|
||||
try (var lines = Files.lines(Path.of(wordsPath), StandardCharsets.UTF_8)) {
|
||||
lines.forEach(line -> CsvIndexService.lineToLemma(line, map::add));
|
||||
return Dicts.makeDict(map.toArray());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to load dictionary from " + wordsPath, e);
|
||||
}
|
||||
}
|
||||
@Test
|
||||
void testExtractSlots() {
|
||||
|
||||
@@ -173,25 +186,35 @@ public class MainTest {
|
||||
}
|
||||
@Test
|
||||
void testFiller2() {
|
||||
val mask = "1 000000\n" +
|
||||
"1 \n" +
|
||||
"1 \n" +
|
||||
"3 3 \n" +
|
||||
"3 0 3 \n" +
|
||||
"3 \n" +
|
||||
"3 \n" +
|
||||
"222 3";
|
||||
val rng = new Rng(-343913721);
|
||||
val mask = Clues.parse(
|
||||
"1 000000\n" +
|
||||
"1 \n" +
|
||||
"1 \n" +
|
||||
"3 3 \n" +
|
||||
"3 0 3 \n" +
|
||||
"3 \n" +
|
||||
"3 \n" +
|
||||
"222 3");
|
||||
Assertions.assertEquals(20, mask.clueCount());
|
||||
var slots = Masker.extractSlots(mask, dict.index());
|
||||
val slotInfo = Masker.scoreSlots(new int[slots.length], slots);
|
||||
var grid = mask.toGrid();
|
||||
var filled = fillMask(rng, slotInfo, grid, false);
|
||||
// val res = new PuzzleResult(new Clued(mask), new Gridded(grid), slotInfo, filled).exportFormatFromFilled(0, new Rewards(0, 0, 0));
|
||||
}
|
||||
@Test
|
||||
void testFiller() {
|
||||
val rng = new Rng(-343913721);
|
||||
val mask = new Clues(
|
||||
74732156493031040L,
|
||||
193L,
|
||||
281475397248512L,
|
||||
128L,
|
||||
422762372923520L,
|
||||
192L);
|
||||
val mask = Clues.parse(
|
||||
" 3 300\n" +
|
||||
" 1 \n" +
|
||||
" 1 \n" +
|
||||
" 3 0 \n" +
|
||||
" 31 \n" +
|
||||
" 1 \n" +
|
||||
" 1 2\n" +
|
||||
"21 22 3");
|
||||
var slots = Masker.extractSlots(mask, dict.index());
|
||||
val slotInfo = Masker.scoreSlots(new int[slots.length], slots);
|
||||
var grid = mask.toGrid();
|
||||
@@ -204,6 +227,7 @@ public class MainTest {
|
||||
var g = new Gridded(grid);
|
||||
g.gridToString(mask);
|
||||
var aa = new PuzzleResult(new Clued(mask), g, slotInfo, filled).exportFormatFromFilled(1, new Rewards(1, 1, 1));
|
||||
System.out.println(String.join("\n", aa.grid()));
|
||||
|
||||
}
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user