introduce bitloops

This commit is contained in:
mike
2026-01-12 00:47:38 +01:00
parent 986c2f85a9
commit 84ba4c9c63
7 changed files with 164 additions and 143 deletions

View File

@@ -8,19 +8,18 @@ import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.Arrays;
import java.util.Locale;
import java.util.function.Consumer;
public final class CsvIndexService
implements Closeable {
static final ScopedValue<CsvIndexService> SC = ScopedValue.newInstance();
static final Gson GSON = new Gson();
private static final int MAGIC = 0x4C494458; // "LIDX"
private static final int VERSION = 1;
private final Path csvPath;
private final Path idxPath;
static final ScopedValue<CsvIndexService> SC = ScopedValue.newInstance();
static final Gson GSON = new Gson();
private static final int MAGIC = 0x4C494458; // "LIDX"
private static final int VERSION = 1;
static int SIMPEL_IDX = 3;
private final Path csvPath;
private final Path idxPath;
private volatile long[] offsets; // lazy
private volatile FileChannel csvChannel; // open once
@@ -30,7 +29,10 @@ public final class CsvIndexService
this.csvPath = csvPath;
this.idxPath = idxPath;
}
public static int lineToSimpel(String line) {
var parts = line.split(",", 5);
return Integer.parseInt(parts[SIMPEL_IDX].trim());
}
public static String[] lineToClue(String line) {
if (line.isBlank()) throw new RuntimeException("Empty line");
var parts = line.split(",", 5);
@@ -51,16 +53,23 @@ public final class CsvIndexService
throw new RuntimeException("Invalid word:" + line);
}
// CSV has level 1-10. llmScores use 10-level.
int score = Integer.parseInt(parts[2].trim());
if (score < 1) {
if (Main.VERBOSE) System.err.println("Word too complex: " + line);
return;
}
int simpel = Integer.parseInt(parts[3].trim());
ok.accept(new Lemma(id, word, simpel));
ok.accept(new Lemma(id, word));
}
public static int simpel(int index) {
try {
if (SC.isBound())
return lineToSimpel(SC.get().getLine(index));
return -1;
} catch (Exception e) {
throw new RuntimeException("Failed to get clues for index " + index, e);
}
}
public static String[] clues(int index) {
try {
if (SC.isBound())